Add support for underscores in numeric literals #2538

compiler/ast/src/value/mod.rs

@@ -877,18 +877,21 @@ impl TryFrom<&Literal> for Value {
             Literal::Group(group_literal) => Self::Group(group_literal.clone()),
             Literal::Scalar(string, span, _) => Self::Scalar(string.clone(), *span),
             Literal::String(string, span, _) => Self::String(string.clone(), *span),
-            Literal::Integer(integer_type, string, span, _) => match integer_type {
-                IntegerType::U8 => Self::U8(string.parse()?, *span),
-                IntegerType::U16 => Self::U16(string.parse()?, *span),
-                IntegerType::U32 => Self::U32(string.parse()?, *span),
-                IntegerType::U64 => Self::U64(string.parse()?, *span),
-                IntegerType::U128 => Self::U128(string.parse()?, *span),
-                IntegerType::I8 => Self::I8(string.parse()?, *span),
-                IntegerType::I16 => Self::I16(string.parse()?, *span),
-                IntegerType::I32 => Self::I32(string.parse()?, *span),
-                IntegerType::I64 => Self::I64(string.parse()?, *span),
-                IntegerType::I128 => Self::I128(string.parse()?, *span),
-            },
+            Literal::Integer(integer_type, raw_string, span, _) => {
+                let string = raw_string.replace('_', "");
+                match integer_type {
+                    IntegerType::U8 => Self::U8(string.parse()?, *span),
+                    IntegerType::U16 => Self::U16(string.parse()?, *span),
+                    IntegerType::U32 => Self::U32(string.parse()?, *span),
+                    IntegerType::U64 => Self::U64(string.parse()?, *span),
+                    IntegerType::U128 => Self::U128(string.parse()?, *span),
+                    IntegerType::I8 => Self::I8(string.parse()?, *span),
+                    IntegerType::I16 => Self::I16(string.parse()?, *span),
+                    IntegerType::I32 => Self::I32(string.parse()?, *span),
+                    IntegerType::I64 => Self::I64(string.parse()?, *span),
+                    IntegerType::I128 => Self::I128(string.parse()?, *span),
+                }
+            }
         })
     }
 }

compiler/parser/src/parser/context.rs

@@ -156,10 +156,15 @@ impl<'a> ParserContext<'a> {
     /// Removes the next token if it is a [`Token::Integer(_)`] and returns it, or [None] if
     /// the next token is not a [`Token::Integer(_)`] or if the next token does not exist.
     ///
-    pub fn eat_integer(&mut self) -> Result<(PositiveNumber, Span)> {
+    pub fn eat_whole_number(&mut self) -> Result<(PositiveNumber, Span)> {
         if let Token::Integer(value) = &self.token.token {
             let value = value.clone();
             self.bump();
+            // Reject value if the length is over 2 and the first character is 0
+            if (value.len() > 1 && value.starts_with('0')) || value.contains('_') {
+                return Err(ParserError::tuple_index_must_be_whole_number(&self.token.token, self.token.span).into());
+            }
+
             Ok((PositiveNumber { value }, self.prev_token.span))
         } else {
             Err(ParserError::unexpected(&self.token.token, "integer literal", self.token.span).into())

compiler/parser/src/parser/expression.rs

@@ -428,7 +428,7 @@ impl ParserContext<'_> {
             if self.eat(&Token::Dot) {
                 if self.check_int() {
                     // Eat a tuple member access.
-                    let (index, span) = self.eat_integer()?;
+                    let (index, span) = self.eat_whole_number()?;
                     expr = Expression::Access(AccessExpression::Tuple(TupleAccess {
                         tuple: Box::new(expr),
                         index,

compiler/parser/src/tokenizer/lexer.rs

@@ -162,7 +162,9 @@ impl Token {
         }
 
         let mut int = String::new();
-        while let Some(c) = input.next_if(|c| c.is_ascii_digit()) {
+
+        // Note that it is still impossible to have a number that starts with an `_` because eat_integer is only called when the first character is a digit.
+        while let Some(c) = input.next_if(|c| c.is_ascii_digit() || *c == '_') {
             if c == '0' && matches!(input.peek(), Some('x')) {
                 int.push(c);
                 int.push(input.next().unwrap());
@@ -264,6 +266,7 @@ impl Token {
                 // + 2 to account for parsing quotation marks.
                 return Ok((string.len() + 2, Token::StaticString(string)));
             }
+
             x if x.is_ascii_digit() => return Self::eat_integer(&mut input),
             '!' => return match_two(&mut input, Token::Not, '=', Token::NotEq),
             '?' => return match_one(&mut input, Token::Question),

compiler/passes/src/type_checking/check_expressions.rs

@@ -588,7 +588,8 @@ impl<'a> ExpressionVisitor<'a> for TypeChecker<'a> {
     }
 
     fn visit_literal(&mut self, input: &'a Literal, expected: &Self::AdditionalInput) -> Self::Output {
-        fn parse_integer_literal<I: FromStr>(handler: &Handler, string: &String, span: Span, type_string: &str) {
+        fn parse_integer_literal<I: FromStr>(handler: &Handler, raw_string: &str, span: Span, type_string: &str) {
+            let string = raw_string.replace('_', "");
             if string.parse::<I>().is_err() {
                 handler.emit_err(TypeCheckerError::invalid_int_value(string, type_string, span));
             }

compiler/passes/src/type_checking/check_statements.rs

@@ -274,6 +274,8 @@ impl<'a> StatementVisitor<'a> for TypeChecker<'a> {
             // Note that this check is needed because the pass attempts to make progress, even though the literal may be invalid.
             if let Ok(value) = Value::try_from(literal) {
                 input.start_value.replace(Some(value));
+            } else {
+                self.emit_err(TypeCheckerError::loop_bound_must_be_a_literal(input.start.span()));
             }
         } else {
             self.emit_err(TypeCheckerError::loop_bound_must_be_a_literal(input.start.span()));
@@ -286,10 +288,30 @@ impl<'a> StatementVisitor<'a> for TypeChecker<'a> {
             // Note that this check is needed because the pass attempts to make progress, even though the literal may be invalid.
             if let Ok(value) = Value::try_from(literal) {
                 input.stop_value.replace(Some(value));
+            } else {
+                self.emit_err(TypeCheckerError::loop_bound_must_be_a_literal(input.stop.span()));
             }
         } else {
             self.emit_err(TypeCheckerError::loop_bound_must_be_a_literal(input.stop.span()));
         }
+
+        // Ensure loop bounds are not decreasing.
+        if match (input.start_value.borrow().as_ref(), input.stop_value.borrow().as_ref()) {
+            (Some(Value::I8(lower_bound, _)), Some(Value::I8(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::I16(lower_bound, _)), Some(Value::I16(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::I32(lower_bound, _)), Some(Value::I32(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::I64(lower_bound, _)), Some(Value::I64(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::I128(lower_bound, _)), Some(Value::I128(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::U8(lower_bound, _)), Some(Value::U8(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::U16(lower_bound, _)), Some(Value::U16(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::U32(lower_bound, _)), Some(Value::U32(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::U64(lower_bound, _)), Some(Value::U64(upper_bound, _))) => lower_bound >= upper_bound,
+            (Some(Value::U128(lower_bound, _)), Some(Value::U128(upper_bound, _))) => lower_bound >= upper_bound,
+            // Note that type mismatch and non-literal errors will already be emitted by here.
+            _ => false,
+        } {
+            self.emit_err(TypeCheckerError::loop_range_decreasing(input.stop.span()));
+        }
     }
 
     fn visit_return(&mut self, input: &'a ReturnStatement) {

errors/src/errors/parser/parser_errors.rs

@@ -283,4 +283,12 @@ create_messages!(
         msg: format!("`console` statements are not yet supported."),
         help: Some("Consider using `assert`, `assert_eq`, or `assert_neq` instead.".to_string()),
     }
+
+    /// Enforce that tuple index must not have leading 0, or underscore in between digits
+    @formatted
+    tuple_index_must_be_whole_number {
+        args: (found: impl Display),
+        msg: format!("expected no underscores or leading zeros -- found '{found}'"),
+        help: None,
+    }
 );

errors/src/errors/type_checker/type_checker_error.rs

@@ -642,4 +642,18 @@ create_messages!(
         msg: format!("This operation can only be used in a `finalize` block."),
         help: None,
     }
+
+    @formatted
+    loop_range_decreasing {
+        args: (),
+        msg: format!("The loop range must be increasing."),
+        help: None,
+    }
+
+    @formatted
+    loop_bound_type_mismatch {
+        args: (),
+        msg: format!("The loop bounds must be same type"),
+        help: None,
+    }
 );

tests/expectations/compiler/integers/i8/add.out

@@ -2,11 +2,11 @@
 namespace: Compile
 expectation: Pass
 outputs:
-  - - initial_ast: 07d84ab17fb71320a01c243bc220b7273b27cd2f4c572b11852afd5128563bb7
-      unrolled_ast: 07d84ab17fb71320a01c243bc220b7273b27cd2f4c572b11852afd5128563bb7
-      ssa_ast: e089fb6b899d91adc9df149257039d771880ff6d31cbcc1c3fcf3223d61e4fcc
-      flattened_ast: a7a814b61f9d3d520375e192824edaf10f378cd65f30746bfcb1e81d4b524940
-      inlined_ast: a7a814b61f9d3d520375e192824edaf10f378cd65f30746bfcb1e81d4b524940
-      dce_ast: a7a814b61f9d3d520375e192824edaf10f378cd65f30746bfcb1e81d4b524940
-      bytecode: 7e5db24495ea3dcca85545d83273ce3c02faae5a2bcaef3a9448920ac68daeda
+  - - initial_ast: 52c17634e4873e8aaed7bc62cbafc7b36a805930fedac25679ea1e44ad68b9d9
+      unrolled_ast: 52c17634e4873e8aaed7bc62cbafc7b36a805930fedac25679ea1e44ad68b9d9
+      ssa_ast: e1b4addbd3d414377d5cac95a487c1d9aca029ddc222dbab08ed00a3d80298d8
+      flattened_ast: 4d5bcd013ddbfa4fe4397ca346b8cbfd74cb0c1f571ac3af4546493550164939
+      inlined_ast: 4d5bcd013ddbfa4fe4397ca346b8cbfd74cb0c1f571ac3af4546493550164939
+      dce_ast: 4d5bcd013ddbfa4fe4397ca346b8cbfd74cb0c1f571ac3af4546493550164939
+      bytecode: b55a8d40426fb145352765c99ed1875c872f2a6a0aeaa46f5734c543b5cc17a0
       warnings: ""

tests/expectations/compiler/statements/block.out

@@ -2,11 +2,11 @@
 namespace: Compile
 expectation: Pass
 outputs:
-  - - initial_ast: 26ccd058cce0c3bd1c9812903f1cc21e8886905964ca565d41782e08631a4722
-      unrolled_ast: 26ccd058cce0c3bd1c9812903f1cc21e8886905964ca565d41782e08631a4722
-      ssa_ast: b99ef5259b4d8c13f7c716d548e5005b0f90291fa128cf5ff2c576a532bcf47d
-      flattened_ast: 29f8729f583503bf96da596bf6308c90a52837bfe47948b19bce1a75ee47efdb
-      inlined_ast: 29f8729f583503bf96da596bf6308c90a52837bfe47948b19bce1a75ee47efdb
-      dce_ast: 29f8729f583503bf96da596bf6308c90a52837bfe47948b19bce1a75ee47efdb
+  - - initial_ast: 1d588d3765da4c9534dbe6f57ec671ff28232b48bfb80c3777799db1267156d5
+      unrolled_ast: 1d588d3765da4c9534dbe6f57ec671ff28232b48bfb80c3777799db1267156d5
+      ssa_ast: 5f52523779c4b3c1e2c05e43d9dba23227b44b52c4c67616454fdc6980a309cb
+      flattened_ast: 99f116e7cab7619853f0481493dabb8044d73980ec3e4f45aecbd231d5bedf0b
+      inlined_ast: 99f116e7cab7619853f0481493dabb8044d73980ec3e4f45aecbd231d5bedf0b
+      dce_ast: 99f116e7cab7619853f0481493dabb8044d73980ec3e4f45aecbd231d5bedf0b
       bytecode: 9f2bbabd0f858db6e5f4e529fdd5e246023994bf27bbabe6dc1aa6bbf8bf5cfd
       warnings: ""

tests/expectations/compiler/statements/iteration_bound_too_large_fail.out

@@ -2,4 +2,4 @@
 namespace: Compile
 expectation: Fail
 outputs:
-  - "Error [ETYC0372008]: The value 1000000000000000000000000000000000000000000000000000000000000000000000000000000000000 is not a valid `u64`\n    --> compiler-test:7:28\n     |\n   7 |         for i:u64 in 0u64..1000000000000000000000000000000000000000000000000000000000000000000000000000000000000u64 {\n     |                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"
+  - "Error [ETYC0372008]: The value 1000000000000000000000000000000000000000000000000000000000000000000000000000000000000 is not a valid `u64`\n    --> compiler-test:7:28\n     |\n   7 |         for i:u64 in 0u64..1000000000000000000000000000000000000000000000000000000000000000000000000000000000000u64 {\n     |                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nError [ETYC0372049]: Loop bound must be a literal.\n    --> compiler-test:7:28\n     |\n   7 |         for i:u64 in 0u64..1000000000000000000000000000000000000000000000000000000000000000000000000000000000000u64 {\n     |                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"

tests/expectations/compiler/statements/loop_decreasing_fail.out

@@ -0,0 +1,5 @@
+---
+namespace: Compile
+expectation: Fail
+outputs:
+  - "Error [ETYC0372078]: The loop range must be increasing.\n    --> compiler-test:7:28\n     |\n   7 |         for i: i8 in 10i8..5i8 {\n     |                            ^^^\n"

tests/expectations/compiler/statements/underscore_for_loop.out

@@ -0,0 +1,12 @@
+---
+namespace: Compile
+expectation: Pass
+outputs:
+  - - initial_ast: 7a88b27e12cbba00a60c6f0d25814df5fbb4d6878a120c1508fc92adac6bb094
+      unrolled_ast: 0742c151a297119b19e1debc977b482fbba534d15f1a6f424e8b88a593d86da7
+      ssa_ast: f25a9fc5ffb10a442d1c343d45c4ab3e2a8a3aead5be07dd4a4109ee4e8dbf43
+      flattened_ast: 960941bf6e20797b225260976a975ab1ee0bb2357f6215168377f094d01c6dba
+      inlined_ast: 960941bf6e20797b225260976a975ab1ee0bb2357f6215168377f094d01c6dba
+      dce_ast: b8851a63f706ce2a4aff3e73653f0f07b5b9ccaa147306baec1bfd997cc5fa9d
+      bytecode: 61cc464cdc1104635ea399648d62a06b112dc3462634b3f992151c6e5572d6f7
+      warnings: ""

tests/expectations/execution/counter.out

@@ -2,13 +2,13 @@
 namespace: Execute
 expectation: Pass
 outputs:
-  - - initial_ast: ff08a4a92839ebe43e5c035ff1ab8991da994f84b167630c9b26cdc8e30028e6
-      unrolled_ast: ff08a4a92839ebe43e5c035ff1ab8991da994f84b167630c9b26cdc8e30028e6
-      ssa_ast: 8fea46feeceac4607d6e09181bce795520d9403f0be606b049225459f10dfe47
-      flattened_ast: 561824a5b5b1291ca6d59dace67780a62c1089536e895830f901c133fa74da85
-      inlined_ast: 561824a5b5b1291ca6d59dace67780a62c1089536e895830f901c133fa74da85
-      dce_ast: 561824a5b5b1291ca6d59dace67780a62c1089536e895830f901c133fa74da85
-      bytecode: f6055195b401bef6fe1e686a256bb743941b1945b7fd4b8f1800aa83dc3b7495
+  - - initial_ast: 437dad4042f19f778819ccccf9964ff9b6b701c2805417ba378d8d5d643d59bc
+      unrolled_ast: 437dad4042f19f778819ccccf9964ff9b6b701c2805417ba378d8d5d643d59bc
+      ssa_ast: 8f85576cdcb97f4b8c348fbad7ab7d85696cbdc1a26a19226903446785db9d20
+      flattened_ast: 83c91f6cefa549e6fc788d063aa22446b095e1b37e8a20d2109c2ba10068230a
+      inlined_ast: 83c91f6cefa549e6fc788d063aa22446b095e1b37e8a20d2109c2ba10068230a
+      dce_ast: 83c91f6cefa549e6fc788d063aa22446b095e1b37e8a20d2109c2ba10068230a
+      bytecode: 18d3fa0f122b8bc035d12ca6fbca2d0d6c923e9ebde740ebf8101b34ee38102a
       warnings: ""
       results:
         dubble: