Fix issue in accept sequence

uiuc-focal-lab · Jan 5, 2025 · e1e258f · e1e258f
1 parent 7aec0f9
commit e1e258f
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 13 deletions.
diff --git a/syncode/dfa_mask_store.py b/syncode/dfa_mask_store.py
@@ -451,12 +451,10 @@ def _lookup_next_tokens(self, dfa_states: Iterable[DFAState], r: ParseResult) ->
                             elif len(accept_sequence) == 2:
                                 overapprox_token_ids |= self._lookup_next_tokens_for_dfa_state(dfa_state, accept_sequence[1])
                             elif len(accept_sequence) == 3:
-                                # This is useful in under-approximating `grammar_strict` mode as they help improve the precision of SynCode
-                                if self._mode == 'grammar_strict':
-                                    # If the DFA state is a final state we can jump to the start of next terminal
-                                    if self._dfas.is_final(dfa_state): 
-                                        ignore_init_state = self._dfas.initial(accept_sequence[1])
-                                        overapprox_token_ids |= self._lookup_next_tokens_for_dfa_state(ignore_init_state, accept_sequence[2])
+                                # If the DFA state is a final state we can jump to the start of next terminal
+                                if self._dfas.is_final(dfa_state): 
+                                    ignore_init_state = self._dfas.initial(accept_sequence[1])
+                                    overapprox_token_ids |= self._lookup_next_tokens_for_dfa_state(ignore_init_state, accept_sequence[2])
                             else:
                                 raise ValueError(f"Invalid accept sequence: {accept_sequence}")
         return overapprox_token_ids

diff --git a/syncode/parse_result.py b/syncode/parse_result.py
@@ -63,12 +63,6 @@ def from_accept_terminals(cur_accept_terminals, next_accept_terminals, remainder
                         accept_sequences.add(AcceptSequence([final_terminal, t2]))
 
                     if ignore_terminals is not None:
-                        # Since ignore terminals are allowed anywhere in the code (final terminal, ignore terminal) is also a valid accept sequence
-                        for tignore in ignore_terminals:
-                            accept_sequences.add(AcceptSequence([final_terminal, tignore]))
-
-                        # These 3 length accept sequences are useful in under-approximating 
-                        # `grammar_strict` mode as they help improve the precision of SynCode
                         for tignore in ignore_terminals:
                             for t2 in next_accept_terminals:
                                 accept_sequences.add(AcceptSequence([final_terminal, tignore, t2]))

diff --git a/tests/test_grammar_go.py b/tests/test_grammar_go.py
@@ -173,7 +173,7 @@ def test_go_parser15(self):
         partial_code = 'package main\n\nimport (\n\t"encoding/json"\n\t"reflect"\n)\nfunc numerical_letter_grade (grades []interface{}) []string {\n\tletter_grades := make([]string, len(grades))\n\tfor i, grade := range grades {\n\t\tswitch grade.('
         res = inc_parser.get_acceptable_next_terminals(partial_code)
         self.assertIn(AcceptSequence(['LPAR', 'TYPE']), res.accept_sequences)
-        self.assertIn(AcceptSequence(['LPAR', '__IGNORE_0']), res.accept_sequences)
+        self.assertIn(AcceptSequence(['LPAR', '__IGNORE_0', 'NAME']), res.accept_sequences)
 
     def test_go_parser16(self):
         inc_parser.reset()