Merge pull request #3 from benniekrijger/fix-tokenizer

Pattern fix in whitespace and punctuation tokenizer
fieg · Apr 15, 2014 · 9492fba · 9492fba
2 parents d2aaa9b + f9e6af8
commit 9492fba
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/Fieg/Bayes/Tokenizer/WhitespaceAndPunctuationTokenizer.php b/src/Fieg/Bayes/Tokenizer/WhitespaceAndPunctuationTokenizer.php
@@ -11,9 +11,11 @@
 
 class WhitespaceAndPunctuationTokenizer implements TokenizerInterface
 {
+    protected $pattern = "/[ ,.?!-:;\\n\\r\\t…_]/";
+
     public function tokenize($string)
     {
-        $retval = preg_split("/[ ,.?!-:\n\t]/i", mb_strtolower($string));
+        $retval = preg_split($this->pattern, mb_strtolower($string));
         $retval = array_filter($retval, 'trim');
         $retval = array_values($retval);