fixed unicode in regexp, fixes #219

.github/workflows/tokentree.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        haxe-version: ['4.3.0', 'nightly']
+        haxe-version: ['4.3.1', 'nightly']
     steps:
     - uses: actions/checkout@v1
     - name: Use Node.js 12
@@ -58,5 +58,5 @@ jobs:
       if: matrix.haxe-version != 'nightly'
       run: npx neko checkstyle/run.n -s src -s test
     - name: Upload results to codecov
-      if: success() && (matrix.haxe-version == '4.2.5')
+      if: success() && (matrix.haxe-version == '4.3.1')
       run: bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"

.haxerc

@@ -1,4 +1,4 @@
 {
-  "version": "4.3.0",
+  "version": "4.3.1",
   "resolveLibs": "scoped"
 }

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## dev branch / next version (1.x.x)
 
+## version 1.2.9 (2023-05-21)
+
+- Fixed unicode in regexp, fixes [#219](https://github.com/HaxeCheckstyle/tokentree/issues/219)
+
 ## version 1.2.8 (2023-02-22)
 
 - Fixed inline call() and inline new handling ([#217](https://github.com/HaxeCheckstyle/tokentree/issues/217))

haxe_libraries/haxeparser.hxml

@@ -1,4 +1,4 @@
-# @install: lix --silent download "gh://github.com/HaxeCheckstyle/haxeparser#f0a7f07101c14dc32b0964dd52af8dcaa322e178" into haxeparser/4.3.0-rc.1/github/f0a7f07101c14dc32b0964dd52af8dcaa322e178
+# @install: lix --silent download "gh://github.com/HaxeCheckstyle/haxeparser#7e98c9aef901b8e26541cf3f8a6e1da0385b237a" into haxeparser/4.3.0-rc.1/github/7e98c9aef901b8e26541cf3f8a6e1da0385b237a
 -lib hxparse
--cp ${HAXE_LIBCACHE}/haxeparser/4.3.0-rc.1/github/f0a7f07101c14dc32b0964dd52af8dcaa322e178/src
+-cp ${HAXE_LIBCACHE}/haxeparser/4.3.0-rc.1/github/7e98c9aef901b8e26541cf3f8a6e1da0385b237a/src
 -D haxeparser=4.3.0-rc.1

haxe_libraries/utest.hxml

@@ -1,5 +1,5 @@
-# @install: lix --silent download "gh://github.com/haxe-utest/utest#5de48a964ca75c8e6321ac9706346a24958af2a4" into utest/1.13.2/github/5de48a964ca75c8e6321ac9706346a24958af2a4
--cp ${HAXE_LIBCACHE}/utest/1.13.2/github/5de48a964ca75c8e6321ac9706346a24958af2a4/src
+# @install: lix --silent download "gh://github.com/haxe-utest/utest#a94f8812e8786f2b5fec52ce9f26927591d26327" into utest/1.13.2/github/a94f8812e8786f2b5fec52ce9f26927591d26327
+-cp ${HAXE_LIBCACHE}/utest/1.13.2/github/a94f8812e8786f2b5fec52ce9f26927591d26327/src
 -D utest=1.13.2
 --macro utest.utils.Macro.checkHaxe()
 --macro utest.utils.Macro.importEnvSettings()

haxelib.json

@@ -7,8 +7,8 @@
     "contributors": [
         "AlexHaxe"
     ],
-    "releasenote": "fixed inline call() and inline new handling - see CHANGELOG for details",
-    "version": "1.2.8",
+    "releasenote": "fixed unicode in regex - see CHANGELOG for details",
+    "version": "1.2.9",
     "url": "https://github.com/HaxeCheckstyle/tokentree",
     "dependencies": {}
 }

package.json

@@ -1,6 +1,6 @@
 {
     "name": "tokentree",
-    "version": "1.2.8",
+    "version": "1.2.9",
     "description": "TokenTree library used by haxe-checkstyle, haxe-formatter and haxe-languageserver",
     "repository": {
         "type": "git",

test.hxml

@@ -8,6 +8,8 @@
 -D unittest
 -x TestMain
 
+-D message-reporting=pretty
+
 -D coverage-console-summary-reporter
 #-D coverage-console-file-summary-reporter
 -D coverage-console-package-summary-reporter

test/tokentree/TokenTreeBuilderParsingTest.hx

@@ -142,6 +142,7 @@ class TokenTreeBuilderParsingTest implements ITest {
 	public function testExpressionLevelCode() {
 		assertCodeParses(CALL_WITHOUT_SEMICOLON);
 		assertCodeParses(ARRAY_ACCESSS_WITHOUT_SEMICOLON);
+		assertCodeParses(UNICODE_REGEX);
 	}
 
 	public function assertCodeParses(code:String, ?pos:PosInfos) {
@@ -1891,4 +1892,8 @@ import #if haxe4 js.lib.Promise #else js.Promise #end as JsPromise;
 	inline public function toString(x:Int, y:Int) {}
 	public inline function toString(x:Int, y:Int) {}
 	";
+
+	var UNICODE_REGEX = "
+		static final ctrlCharacters = ~/[\\u0000-\\u001F\\u007F-\\u009F\\u2000-\\u200D\\uFEFF]/g;
+	";
 }