/**
* Split the given input into tokens using whitespace as splitter
*
* The input can be a string or a tokenRegistry. If the input is a
* TokenRegistry, each item will be tokenized.
*
* @param string|\Org\Heigl\Hyphenator\Tokenizer\TokenRegistry $input The
* input to be tokenized
*
* @return \Org\Heigl\Hyphenator\Tokenizer\TokenRegistry
*/
public function run($input)
{
if ($input instanceof TokenRegistry) {
// Tokenize a TokenRegistry
foreach ($input as $token) {
if ($token instanceof WhitespaceToken) {
continue;
}
$newTokens = $this->_tokenize($token->get());
if ($newTokens == array($token)) {
continue;
}
$input->replace($token, $newTokens);
}
return $input;
}
// Tokenize a simple string.
$array = $this->_tokenize($input);
$registry = new TokenRegistry();
foreach ($array as $item) {
$registry->add($item);
}
return $registry;
}
/**
* Pass the given string through the given tokenizers
*
* @param string $string The String to be tokenized
*
* @return \Org\Heigl\Hyphenator\TokenRegistry
*/
public function tokenize($string)
{
if (!$string instanceof TokenRegistry) {
$wt = new WordToken($string);
$string = new TokenRegistry();
$string->add($wt);
}
foreach ($this as $tokenizer) {
$string = $tokenizer->run($string);
}
return $string;
}
