From 9a20a97e5c9626a3ae79aaba64030524574ac6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=20De=20Caluw=C3=A9?= Date: Wed, 11 Nov 2020 01:44:35 -0500 Subject: [PATCH] Add Tokenizer test for bug #33. --- spec/TokenizerSpec.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 spec/TokenizerSpec.js diff --git a/spec/TokenizerSpec.js b/spec/TokenizerSpec.js new file mode 100644 index 0000000..84c4c60 --- /dev/null +++ b/spec/TokenizerSpec.js @@ -0,0 +1,35 @@ +'use strict' + +const Tokenizer = require('../tokenizer.js'); +const Configuration = require('../configuration.js'); +const Cache = require('../cache.js'); + +describe('Tokenizer', function () { + let tokenizer; + let configuration; + + beforeEach(function () { + Tokenizer.cache = new Cache(10); + configuration = new Configuration(); + tokenizer = new Tokenizer(configuration); + }); + it('should stop tokenizing at special characters', function () { + tokenizer.data('TEST DATA+', 0); + expect(tokenizer.content()).toEqual('TEST DATA'); + }); + it('should properly initialize when using cached regexes', function () { + // This test was written specifically for a bug where the default regex + // wasn't set after a configuration with a cache hit. + var cached = new Tokenizer(configuration); + var character = String.fromCharCode(50); + + configuration = new Configuration({ decimalMark: 50 }); + + tokenizer.configure(configuration); + cached.configure(configuration); + tokenizer.data(character, 0); + cached.data(character, 0); + + expect(tokenizer.length).toEqual(cached.length); + }); +}); \ No newline at end of file