README.md

Tokenize

Tokenizing text

use text\{StringTokenizer, StreamTokenizer};
use io\File;

// Supports strings and streams
$tokens= new StringTokenizer('He asked: Can you parse this?', ' .?!,;:', true);
$tokens= new StreamTokenizer((new File('parse-me.txt'))->in(), ' .?!,;:', true);

// Can iterate using foreach...
foreach ($tokens as $token) {
  Console::writeLine($token);
}

// ...or with an iterator API
while ($tokens->hasMoreTokens()) {
  Console::writeLine($tokens->nextToken());
}

// Returns: ["He", " ", "asked", ":", " ", "Can", " ", "you", " ", "parse", " ", "this", "?"]