From 46231e5f7d68d03dd5775e01286f0bbcf4310ca2 Mon Sep 17 00:00:00 2001 From: brandon-gong Date: Sat, 6 Nov 2021 22:51:47 -0400 Subject: [PATCH 1/2] Fix tokenizeFrom string length changes on normalize + add tests --- lib/jglr/jglr.js | 2 +- tests/pyret/tests/test-parse.arr | 3 ++- tests/pyret/tests/test-strings.arr | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/jglr/jglr.js b/lib/jglr/jglr.js index c8ed9bfbd0..0ee432db08 100644 --- a/lib/jglr/jglr.js +++ b/lib/jglr/jglr.js @@ -216,7 +216,7 @@ define("jglr/jglr", ["jglr/rnglr"], function(E) { else this.str = str; - this.len = str.length; // XXX Not necessarily unicode-aware + this.len = this.str.length; // XXX Not necessarily unicode-aware this.curCol = 0; this.curLine = 1; this.pos = 0; diff --git a/tests/pyret/tests/test-parse.arr b/tests/pyret/tests/test-parse.arr index e8d529beed..7751ba4f19 100644 --- a/tests/pyret/tests/test-parse.arr +++ b/tests/pyret/tests/test-parse.arr @@ -389,6 +389,7 @@ check "should parse block comments": does-parse('lam(x #| stuff |#, y): x + y end') is true does-parse('lam(x #| two |##| comments|#, y): x + y end') is true does-parse('#| |# |#') is false + does-parse('#|क़ string length changes on normalize|#') is true end check "should not ignore the line after an empty hash comment": @@ -604,4 +605,4 @@ check "should parse use-stmt": use b("c") ```) is false -end \ No newline at end of file +end diff --git a/tests/pyret/tests/test-strings.arr b/tests/pyret/tests/test-strings.arr index 2fdced6acb..718fb12624 100644 --- a/tests/pyret/tests/test-strings.arr +++ b/tests/pyret/tests/test-strings.arr @@ -44,6 +44,7 @@ check: string-length("a") is 1 string-length("\n\r \t") is 4 string-length("λjs") is 3 + string-length("𢡊") is 2 string-tonumber("45") is 45 # TODO(joe): some string-to-number parsing issue on the number below From a0d327f69bed4ac205a99608431c58e3af41ddbd Mon Sep 17 00:00:00 2001 From: brandon-gong Date: Sat, 6 Nov 2021 23:00:45 -0400 Subject: [PATCH 2/2] fix mismatched whitespace in tests --- tests/pyret/tests/test-parse.arr | 2 +- tests/pyret/tests/test-strings.arr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pyret/tests/test-parse.arr b/tests/pyret/tests/test-parse.arr index 7751ba4f19..4c6c51ea84 100644 --- a/tests/pyret/tests/test-parse.arr +++ b/tests/pyret/tests/test-parse.arr @@ -389,7 +389,7 @@ check "should parse block comments": does-parse('lam(x #| stuff |#, y): x + y end') is true does-parse('lam(x #| two |##| comments|#, y): x + y end') is true does-parse('#| |# |#') is false - does-parse('#|क़ string length changes on normalize|#') is true + does-parse('#|क़ string length changes on normalize|#') is true end check "should not ignore the line after an empty hash comment": diff --git a/tests/pyret/tests/test-strings.arr b/tests/pyret/tests/test-strings.arr index 718fb12624..bec5b52551 100644 --- a/tests/pyret/tests/test-strings.arr +++ b/tests/pyret/tests/test-strings.arr @@ -44,7 +44,7 @@ check: string-length("a") is 1 string-length("\n\r \t") is 4 string-length("λjs") is 3 - string-length("𢡊") is 2 + string-length("𢡊") is 2 string-tonumber("45") is 45 # TODO(joe): some string-to-number parsing issue on the number below