SONARPY-1538: Fix lexing error when encountering escaped characters (#…

…1625)
SonarSource · Oct 30, 2023 · 09ede81 · 09ede81
1 parent 1ebb12e
commit 09ede81
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/python-frontend/src/main/java/org/sonar/python/lexer/FStringChannel.java b/python-frontend/src/main/java/org/sonar/python/lexer/FStringChannel.java
@@ -47,7 +47,7 @@ public class FStringChannel extends Channel<Lexer> {
 
   private static final Set<Character> QUOTES = Set.of('\"', '\'');
   private static final Set<Character> PREFIXES = Set.of('F', 'R');
-  private static final Set<String> ESCAPED_CHARS = Set.of("{{", "}}","\\\"","\\\'");
+  private static final Set<String> ESCAPED_CHARS = Set.of("{{", "}}");
 
   public FStringChannel(LexerState lexerState) {
     this.lexerState = lexerState;
@@ -153,7 +153,7 @@ private static boolean isUnicodeChar(StringBuilder sb ){
   }
 
   private static boolean isEscapedChar(CodeReader code) {
-    return ESCAPED_CHARS.contains(String.valueOf(code.peek(2)));
+    return ESCAPED_CHARS.contains(String.valueOf(code.peek(2))) || code.peek() == '\\';
   }
 
   private static boolean areClosingQuotes(CodeReader code, FStringState state) {

diff --git a/python-frontend/src/test/java/org/sonar/python/lexer/PythonLexerTest.java b/python-frontend/src/test/java/org/sonar/python/lexer/PythonLexerTest.java
@@ -527,6 +527,17 @@ void fstring_complex_format_specifier() {
       hasToken("}", PythonPunctuator.RCURLYBRACE),
       hasToken("\"", PythonTokenType.FSTRING_END)));
   }
+
+  @Test
+  void fstring_double_backslash() {
+    assertThat(lexer.lex("f\"{a}\\\\\""), allOf(
+      hasToken("f\"", PythonTokenType.FSTRING_START),
+      hasToken("{", PythonPunctuator.LCURLYBRACE),
+      hasToken("a", GenericTokenType.IDENTIFIER),
+      hasToken("}", PythonPunctuator.RCURLYBRACE),
+      hasToken("\\\\", PythonTokenType.FSTRING_MIDDLE),
+      hasToken("\"", PythonTokenType.FSTRING_END)));
+  }
   /**
    * http://docs.python.org/reference/lexical_analysis.html#integer-and-long-integer-literals
    */

diff --git a/python-frontend/src/test/resources/parser/own/fstring.py b/python-frontend/src/test/resources/parser/own/fstring.py
@@ -24,4 +24,5 @@
     }"
 f"Current value: \"{value}\" (type: {type(value)}). "
 f'\N{RIGHTWARDS ARROW}'
-
+f" \\"
+F"\\ \"{a}\":\\"