css2: Support starting idents w/ escaped code points

css2/tokenizer.cpp

@@ -41,6 +41,11 @@ constexpr bool is_digit(std::optional<char> c) {
     return c && util::is_digit(*c);
 }
 
+// https://www.w3.org/TR/css-syntax-3/#check-if-two-code-points-are-a-valid-escape
+constexpr bool is_valid_escape_sequence(char first_byte, std::optional<char> second_byte) {
+    return first_byte == '\\' && second_byte != '\n';
+}
+
 } // namespace
 
 std::string_view to_string(ParseError e) {
@@ -79,6 +84,29 @@ void Tokenizer::run() {
                         current_token_ = StringToken{""};
                         state_ = State::String;
                         continue;
+                    case '#': {
+                        auto next_input = peek_input(0);
+                        if (!next_input) {
+                            emit(DelimToken{'#'});
+                            continue;
+                        }
+
+                        if (is_ident_code_point(*next_input) || is_valid_escape_sequence(*next_input, peek_input(1))) {
+                            std::ignore = consume_next_input_character();
+                            HashToken token{};
+
+                            if (inputs_starts_ident_sequence(*next_input)) {
+                                token.type = HashToken::Type::Id;
+                            }
+
+                            token.data = consume_an_ident_sequence(*next_input);
+                            emit(std::move(token));
+                            continue;
+                        }
+
+                        emit(DelimToken{'#'});
+                        continue;
+                    }
                     case '/':
                         state_ = State::CommentStart;
                         continue;
@@ -330,18 +358,25 @@ std::optional<char> Tokenizer::peek_input(int index) const {
 
 // https://www.w3.org/TR/css-syntax-3/#would-start-an-identifier
 bool Tokenizer::inputs_starts_ident_sequence(char first_character) const {
-    bool result{false};
     if (first_character == '-') {
-        if (auto second_character = peek_input(0)) {
-            if (is_ident_start_code_point(*second_character) || *second_character == '-') {
-                result = true;
-            }
+        auto second_character = peek_input(0);
+        if (!second_character) {
+            return false;
+        }
+
+        if (is_ident_start_code_point(*second_character) || *second_character == '-') {
+            return true;
         }
-    } else if (is_ident_start_code_point(first_character)) {
-        result = true;
+
+        auto third_character = peek_input(1);
+        return is_valid_escape_sequence(*second_character, third_character);
     }
-    // TODO(mkiael): Handle escape sequence
-    return result;
+
+    if (is_ident_start_code_point(first_character)) {
+        return true;
+    }
+
+    return is_valid_escape_sequence(first_character, peek_input(0));
 }
 
 bool Tokenizer::inputs_starts_number([[maybe_unused]] char first_character) const {
@@ -529,20 +564,19 @@ Token Tokenizer::consume_a_numeric_token(char first_byte) {
 
 // https://www.w3.org/TR/css-syntax-3/#consume-name
 std::string Tokenizer::consume_an_ident_sequence(char first_byte) {
-    std::string result{first_byte};
-    while (auto c = peek_input(0)) {
+    std::string result{};
+    for (std::optional<char> c = first_byte; c.has_value(); c = consume_next_input_character()) {
         if (is_ident_code_point(*c)) {
-            std::ignore = consume_next_input_character();
             result += *c;
             continue;
         }
 
         if (*c == '\\') {
-            std::ignore = consume_next_input_character();
             result += consume_an_escaped_code_point();
             continue;
         }
 
+        reconsume();
         break;
     }
 

css2/tokenizer_test.cpp

@@ -303,6 +303,23 @@ int main() {
         expect_token(output, AtKeywordToken{"foo#"});
     });
 
+    s.add_test("at keyword starting w/ an escape", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "@\\23 bc");
+        expect_token(output, AtKeywordToken{"#bc"});
+    });
+
+    s.add_test("at keyword starting w/ - + escape", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "@-\\23 bc");
+        expect_token(output, AtKeywordToken{"-#bc"});
+    });
+
+    s.add_test("at keyword start, but with bad escape", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "@\\\n");
+        expect_token(output, DelimToken{'@'});
+        expect_token(output, DelimToken{'\\'});
+        expect_token(output, WhitespaceToken{});
+    });
+
     s.add_test("at keyword token with digit", [](etest::IActions &a) {
         auto output = run_tokenizer(a, "@b4z");
 
@@ -586,5 +603,32 @@ int main() {
         expect_token(output, CloseParenToken{});
     });
 
+    s.add_test("hash token: ez", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "#");
+        expect_token(output, DelimToken{'#'});
+    });
+
+    s.add_test("hash token: ident sequence", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "#foo");
+        expect_token(output, HashToken{.type = HashToken::Type::Id, .data = "foo"});
+    });
+
+    s.add_test("hash token: non-ident sequence", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "#123");
+        expect_token(output, HashToken{.type = HashToken::Type::Unrestricted, .data = "123"});
+    });
+
+    s.add_test("hash token: escaped code point", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "#\\41");
+        expect_token(output, HashToken{.type = HashToken::Type::Id, .data = "A"});
+    });
+
+    s.add_test("hash token: invalid escape", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "#\\\n");
+        expect_token(output, DelimToken{'#'});
+        expect_token(output, DelimToken{'\\'});
+        expect_token(output, WhitespaceToken{});
+    });
+
     return s.run();
 }