- lexer.rl: handle CLRF as a line separator

lib/parser/lexer.rl

@@ -545,7 +545,7 @@ class Parser::Lexer
     @newline_s = p
   }
 
-  c_nl       = '\n' $ do_nl;
+  c_nl       = ('\r\n' | '\n') $ do_nl;
   c_space    = [ \t\r\f\v];
   c_space_nl = c_space | c_nl;
 
@@ -1995,7 +1995,7 @@ class Parser::Lexer
 
       # Here we use '\n' instead of w_newline to not modify @newline_s
       # and eventually properly emit tNL
-      (c_space* w_space_comment '\n')+
+      (c_space* w_space_comment ('\r\n' | '\n'))+
       => {
         if @version < 27
           # Ruby before 2.7 doesn't support comments before leading dot.

lib/parser/lexer/literal.rb

@@ -208,7 +208,7 @@ def flush_string
       end
 
       unless @buffer.empty?
-        emit(:tSTRING_CONTENT, @buffer, @buffer_s, @buffer_e)
+        emit(:tSTRING_CONTENT, @buffer.gsub("\r\n", "\n"), @buffer_s, @buffer_e)
 
         clear_buffer
         extend_content

lib/parser/source/buffer.rb

@@ -187,7 +187,7 @@ def raw_source=(input)
           raise ArgumentError, 'Source::Buffer is immutable'
         end
 
-        @source = input.gsub("\r\n".freeze, "\n".freeze).freeze
+        @source = input.freeze
 
         if [email protected]_only? &&
            @source.encoding != Encoding::UTF_32LE &&

test/test_lexer.rb

@@ -1012,9 +1012,9 @@ def test_heredoc_cr
                    :tIDENTIFIER,     "a",          [0, 1],
                    :tEQL,            "=",          [2, 3],
                    :tSTRING_BEG,     "<<\"",       [4, 7],
-                   :tSTRING_CONTENT, "ABCDEF\r\n", [9, 17],
-                   :tSTRING_END,     "E",          [17, 20],
-                   :tNL,             nil,          [8, 9])
+                   :tSTRING_CONTENT, "ABCDEF\r\n", [10, 19],
+                   :tSTRING_END,     "E",          [19, 23],
+                   :tNL,             nil,          [9, 10])
   end
 
   def test_heredoc_with_identifier_ending_newline__19
@@ -3188,9 +3188,9 @@ def test_bug_heredoc_continuation
   def test_bug_heredoc_cr_lf
     assert_scanned("<<FIN\r\nfoo\r\nFIN\r\n",
                    :tSTRING_BEG,     "<<\"",  [0, 5],
-                   :tSTRING_CONTENT, "foo\n", [6, 10],
-                   :tSTRING_END,     "FIN",   [10, 13],
-                   :tNL,             nil,     [5, 6])
+                   :tSTRING_CONTENT, "foo\n", [7, 12],
+                   :tSTRING_END,     "FIN",   [12, 16],
+                   :tNL,             nil,     [6, 7])
   end
 
   def test_bug_eh_symbol_no_newline

test/test_parser.rb

@@ -5610,7 +5610,7 @@ def test_crlf_line_endings
       assert_equal s(:lvar, :foo),
                    ast
 
-      assert_equal range.call(1, 4),
+      assert_equal range.call(2, 5),
                    ast.loc.expression
     end
   end