Repository: ruby/prism
Pull request #2874 (2024-06-07 19:24:37)

Heredoc newlines in parser translation


Patch:
diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb
index c6205e54..48f3d4db 100644
--- a/lib/prism/translation/parser/compiler.rb
+++ b/lib/prism/translation/parser/compiler.rb
@@ -2040,6 +2040,13 @@ module Prism
           end
         end
 
+        # The parser gem automatically converts \r\n to \n, meaning our offsets
+        # need to be adjusted to always subtract 1 from the length.
+        def chomped_bytesize(line)
+          chomped = line.chomp
+          chomped.bytesize + (chomped == line ? 0 : 1)
+        end
+
         # Visit a heredoc that can be either a string or an xstring.
         def visit_heredoc(node)
           children = Array.new
@@ -2057,23 +2064,31 @@ module Prism
           node.parts.each do |part|
             pushing =
               if part.is_a?(StringNode) && part.unescaped.include?("\n")
-                unescaped = part.unescaped.lines(chomp: true)
-                escaped = part.content.lines(chomp: true)
+                unescaped = part.unescaped.lines
+                escaped = part.content.lines
 
-                escaped_lengths =
-                  if node.opening.end_with?("'")
-                    escaped.map { |line| line.bytesize + 1 }
-                  else
-                    escaped.chunk_while { |before, after| before.match?(/(?<!\\)\\$/) }.map { |line| line.join.bytesize + line.length }
+                escaped_lengths = []
+                normalized_lengths = []
+
+                if node.opening.end_with?("'")
+                  escaped.each do |line|
+                    escaped_lengths << line.bytesize
+                    normalized_lengths << chomped_bytesize(line)
                   end
+                else
+                  escaped
+                    .chunk_while { |before, after| before.match?(/(?<!\\)\\\r?\n$/) }
+                    .each do |lines|
+                      escaped_lengths << lines.sum(&:bytesize)
+                      normalized_lengths << lines.sum { |line| chomped_bytesize(line) }
+                    end
+                end
 
                 start_offset = part.location.start_offset
-                end_offset = nil
 
-                unescaped.zip(escaped_lengths).map do |unescaped_line, escaped_length|
-                  end_offset = start_offset + (escaped_length || 0)
-                  inner_part = builder.string_internal(["#{unescaped_line}\n", srange_offsets(start_offset, end_offset)])
-                  start_offset = end_offset
+                unescaped.map.with_index do |unescaped_line, index|
+                  inner_part = builder.string_internal([unescaped_line, srange_offsets(start_offset, start_offset + normalized_lengths.fetch(index, 0))])
+                  start_offset += escaped_lengths.fetch(index, 0)
                   inner_part
                 end
               else
@@ -2084,7 +2099,12 @@ module Prism
               if child.type == :str && child.children.last == ""
                 # nothing
               elsif child.type == :str && children.last && children.last.type == :str && !children.last.children.first.end_with?("\n")
-                children.last.children.first << child.children.first
+                appendee = children[-1]
+
+                location = appendee.loc
+                location = location.with_expression(location.expression.join(child.loc.expression))
+
+                children[-1] = appendee.updated(:str, [appendee.children.first << child.children.first], location: location)
               else
                 children << child
               end


Test patch:
diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb
index 61ea7ae6..65535af0 100644
--- a/test/prism/ruby/parser_test.rb
+++ b/test/prism/ruby/parser_test.rb
@@ -49,6 +49,9 @@ module Prism
       "spanning_heredoc.txt",
       "spanning_heredoc_newlines.txt",
 
+      # https://github.com/whitequark/parser/issues/1021
+      "seattlerb/heredoc_nested.txt",
+
       # https://github.com/whitequark/parser/issues/1016
       "whitequark/unary_num_pow_precedence.txt"
     ]
@@ -56,15 +59,10 @@ module Prism
     # These files are either failing to parse or failing to translate, so we'll
     # skip them for now.
     skip_all = skip_incorrect | [
-      "dash_heredocs.txt",
-      "heredocs_with_ignored_newlines.txt",
       "regex.txt",
       "regex_char_width.txt",
       "unescaping.txt",
       "seattlerb/bug190.txt",
-      "seattlerb/heredoc_nested.txt",
-      "seattlerb/heredoc_with_carriage_return_escapes_windows.txt",
-      "seattlerb/heredoc_with_carriage_return_escapes.txt",
       "seattlerb/heredoc_with_extra_carriage_returns_windows.txt",
       "seattlerb/heredoc_with_only_carriage_returns_windows.txt",
       "seattlerb/heredoc_with_only_carriage_returns.txt",
@@ -74,10 +72,8 @@ module Prism
       "unparser/corpus/literal/literal.txt",
       "unparser/corpus/semantic/dstr.txt",
       "whitequark/dedenting_interpolating_heredoc_fake_line_continuation.txt",
-      "whitequark/parser_bug_640.txt",
       "whitequark/parser_slash_slash_n_escaping_in_literals.txt",
-      "whitequark/ruby_bug_11989.txt",
-      "whitequark/slash_newline_in_heredocs.txt"
+      "whitequark/ruby_bug_11989.txt"
     ]
 
     # Not sure why these files are failing on JRuby, but skipping them for now.
@@ -89,9 +85,11 @@ module Prism
     # output expected by the parser gem, so we'll skip them for now.
     skip_tokens = [
       "comments.txt",
+      "dash_heredocs.txt",
       "dos_endings.txt",
       "embdoc_no_newline_at_end.txt",
       "heredoc_with_comment.txt",
+      "heredocs_with_ignored_newlines.txt",
       "indented_file_end.txt",
       "methods.txt",
       "strings.txt",
@@ -116,6 +114,8 @@ module Prism
       "seattlerb/heredoc_squiggly_visually_blank_lines.txt",
       "seattlerb/heredoc_squiggly.txt",
       "seattlerb/heredoc_unicode.txt",
+      "seattlerb/heredoc_with_carriage_return_escapes_windows.txt",
+      "seattlerb/heredoc_with_carriage_return_escapes.txt",
       "seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt",
       "seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt",
       "seattlerb/interpolated_symbol_array_line_breaks.txt",
@@ -163,10 +163,12 @@ module Prism
       "whitequark/lbrace_arg_after_command_args.txt",
       "whitequark/multiple_pattern_matches.txt",
       "whitequark/newline_in_hash_argument.txt",
+      "whitequark/parser_bug_640.txt",
       "whitequark/parser_drops_truncated_parts_of_squiggly_heredoc.txt",
       "whitequark/ruby_bug_11990.txt",
       "whitequark/ruby_bug_14690.txt",
       "whitequark/ruby_bug_9669.txt",
+      "whitequark/slash_newline_in_heredocs.txt",
       "whitequark/space_args_arg_block.txt",
       "whitequark/space_args_block.txt"
     ]