Repaired sentence tokenizer for braces and quotes (issue #591) (#592)

NaturalNode · Mar 24, 2021 · 007f455 · 007f455
1 parent 2a8cac7
commit 007f455
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 1 deletion.
diff --git a/lib/natural/tokenizers/sentence_tokenizer.js b/lib/natural/tokenizers/sentence_tokenizer.js
@@ -34,7 +34,7 @@ util.inherits(SentenceTokenizer, Tokenizer)
 
 SentenceTokenizer.prototype.tokenize = function (text) {
   // break string up in to sentences based on punctation and quotation marks
-  let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨](.*?[.?!])(\s[.?!])*["'’”'"\])}⟩](?=\s+|$)|(?<=\s+|^)\S(.*?[.?!])(\s[.?!])*(?=\s+|$)/g)
+  let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?(.*?[.?!])(\s[.?!])*["'’”'"\])}⟩]?(?=\s+|$)|(?<=\s+|^)\S(.*?[.?!])(\s[.?!])*(?=\s+|$)/g)
 
   DEBUG && console.log('SentenceTokenizer.tokenize: ' + tokens)
 

diff --git a/spec/sentence_tokenizer_spec.js b/spec/sentence_tokenizer_spec.js
@@ -131,4 +131,31 @@ describe('sentence_tokenizer', function () {
       'Same Bat-Channel!'
     ])
   })
+
+  it('should handle braces and quotes (issue #591)', function () {
+    expect(
+      tokenizer.tokenize('Teste. Test test. Test test: “Test.”')
+    ).toEqual([
+      'Teste.',
+      'Test test.',
+      'Test test: “Test.”'
+    ])
+    expect(
+      tokenizer.tokenize('Test Test. Test test, test test (test test) test: “Test.”')
+    ).toEqual([
+      'Test Test.',
+      'Test test, test test (test test) test: “Test.”'
+    ])
+    expect(
+      tokenizer.tokenize('Test Test. Test test, test (test) test (test test) test: “Test.”')
+    ).toEqual([
+      'Test Test.',
+      'Test test, test (test) test (test test) test: “Test.”'
+    ])
+    expect(
+      tokenizer.tokenize('Test: Test (test) test “Test.”')
+    ).toEqual([
+      'Test: Test (test) test “Test.”'
+    ])
+  })
 })