Fix UTF-8 decoding on 4-byte characters (#712)

HigherOrderCO · Sep 10, 2024 · d91742e · d91742e
1 parent a5054e4
commit d91742e
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 1 deletion.
diff --git a/src/fun/builtins.bend b/src/fun/builtins.bend
@@ -498,7 +498,7 @@ Utf8/decode_character (List/Cons a (List/Cons b (List/Cons c (List/Cons d rest))
       } else {
         if (== (& a 0xF8) 0xF0) {
           let r = (| (<< (& a Utf8/mask4) 18) (| (<< (& b Utf8/maskx) 12) (| (<< (& c Utf8/maskx) 6) (& d Utf8/maskx))))
-          (r, [])
+          (r, rest)
         } else {
           (Utf8/REPLACEMENT_CHARACTER, rest)
         }

diff --git a/tests/golden_tests/io/utf8.bend b/tests/golden_tests/io/utf8.bend
@@ -0,0 +1,9 @@
+to-and-back s = (String/decode_utf8 (String/encode_utf8 s))
+
+v1 = (to-and-back "hi")
+v2 = (to-and-back "(λf ((λx (f (x x))) (λx (f (x x)))))")
+v3 = (to-and-back "🌟")
+v4 = (to-and-back "Hello 🌎!")
+v5 = (to-and-back "𓆈 𓆉 𓆊 𓆋 𓅯")
+
+main = [v1, v2, v3, v4, v5]
diff --git a/tests/snapshots/io__utf8.bend.snap b/tests/snapshots/io__utf8.bend.snap
@@ -0,0 +1,6 @@
+---
+source: tests/golden_tests.rs
+input_file: tests/golden_tests/io/utf8.bend
+---
+Strict mode:
+["hi", "(λf ((λx (f (x x))) (λx (f (x x)))))", "🌟", "Hello 🌎!", "𓆈 𓆉 𓆊 𓆋 𓅯"]