Repository: ruby/prism
Pull request #2853 (2024-05-31 19:12:02)

Parser translation work


Patch:
diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb
index 08e54a7b..816c8841 100644
--- a/lib/prism/translation/parser/compiler.rb
+++ b/lib/prism/translation/parser/compiler.rb
@@ -90,7 +90,11 @@ module Prism
           end
 
           if node.constant
-            builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(nil, visited, nil), token(node.closing_loc))
+            if visited.empty?
+              builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(token(node.opening_loc), visited, token(node.closing_loc)), token(node.closing_loc))
+            else
+              builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(nil, visited, nil), token(node.closing_loc))
+            end
           else
             builder.array_pattern(token(node.opening_loc), visited, token(node.closing_loc))
           end
@@ -106,14 +110,20 @@ module Prism
         #   ^^^^
         def visit_assoc_node(node)
           if in_pattern
+            key = node.key
+
             if node.value.is_a?(ImplicitNode)
-              if node.key.is_a?(SymbolNode)
-                builder.match_hash_var([node.key.unescaped, srange(node.key.location)])
+              if key.is_a?(SymbolNode)
+                if key.opening.nil?
+                  builder.match_hash_var([key.unescaped, srange(key.location)])
+                else
+                  builder.match_hash_var_from_str(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.value_loc)])], token(key.closing_loc))
+                end
               else
-                builder.match_hash_var_from_str(token(node.key.opening_loc), visit_all(node.key.parts), token(node.key.closing_loc))
+                builder.match_hash_var_from_str(token(key.opening_loc), visit_all(key.parts), token(key.closing_loc))
               end
             else
-              builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value))
+              builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value))
             end
           elsif node.value.is_a?(ImplicitNode)
             if (value = node.value.value).is_a?(LocalVariableReadNode)
@@ -146,7 +156,9 @@ module Prism
         # { **foo }
         #   ^^^^^
         def visit_assoc_splat_node(node)
-          if node.value.nil? && forwarding.include?(:**)
+          if in_pattern
+            builder.match_rest(token(node.operator_loc), token(node.value&.location))
+          elsif node.value.nil? && forwarding.include?(:**)
             builder.forwarded_kwrestarg(token(node.operator_loc))
           else
             builder.kwsplat(token(node.operator_loc), visit(node.value))
@@ -1064,36 +1076,7 @@ module Prism
         # ^^^^^^^^^^^^
         def visit_interpolated_string_node(node)
           if node.heredoc?
-            children, closing = visit_heredoc(node)
-            opening = token(node.opening_loc)
-
-            start_offset = node.opening_loc.end_offset + 1
-            end_offset = node.parts.first.location.start_offset
-
-            # In the below case, the offsets should be the same:
-            #
-            # <<~HEREDOC
-            #   a #{b}
-            # HEREDOC
-            #
-            # But in this case, the end_offset would be greater than the start_offset:
-            #
-            # <<~HEREDOC
-            #   #{b}
-            # HEREDOC
-            #
-            # So we need to make sure the result node's heredoc range is correct, without updating the children
-            result = if start_offset < end_offset
-              # We need to add a padding string to ensure that the heredoc has correct range for its body
-              padding_string_node = builder.string_internal(["", srange_offsets(start_offset, end_offset)])
-              node_with_correct_location = builder.string_compose(opening, [padding_string_node, *children], closing)
-              # But the padding string should not be included in the final AST, so we need to update the result's children
-              node_with_correct_location.updated(:dstr, children)
-            else
-              builder.string_compose(opening, children, closing)
-            end
-
-            return result
+            return visit_heredoc(node) { |children, closing| builder.string_compose(token(node.opening_loc), children, closing) }
           end
 
           parts = if node.parts.one? { |part| part.type == :string_node }
@@ -1137,8 +1120,7 @@ module Prism
         # ^^^^^^^^^^^^
         def visit_interpolated_x_string_node(node)
           if node.heredoc?
-            children, closing = visit_heredoc(node)
-            builder.xstring_compose(token(node.opening_loc), children, closing)
+            visit_heredoc(node) { |children, closing| builder.xstring_compose(token(node.opening_loc), children, closing) }
           else
             builder.xstring_compose(
               token(node.opening_loc),
@@ -1673,10 +1655,11 @@ module Prism
         # ^^^^^
         def visit_string_node(node)
           if node.heredoc?
-            children, closing = visit_heredoc(node.to_interpolated)
-            builder.string_compose(token(node.opening_loc), children, closing)
+            visit_heredoc(node.to_interpolated) { |children, closing| builder.string_compose(token(node.opening_loc), children, closing) }
           elsif node.opening == "?"
             builder.character([node.unescaped, srange(node.location)])
+          elsif node.opening&.start_with?("%") && node.unescaped.empty?
+            builder.string_compose(token(node.opening_loc), [], token(node.closing_loc))
           else
             content_lines = node.content.lines
             unescaped_lines = node.unescaped.lines
@@ -1876,8 +1859,7 @@ module Prism
         # ^^^^^
         def visit_x_string_node(node)
           if node.heredoc?
-            children, closing = visit_heredoc(node.to_interpolated)
-            builder.xstring_compose(token(node.opening_loc), children, closing)
+            visit_heredoc(node.to_interpolated) { |children, closing| builder.xstring_compose(token(node.opening_loc), children, closing) }
           else
             parts = if node.unescaped.lines.one?
               [builder.string_internal([node.unescaped, srange(node.content_loc)])]
@@ -2041,6 +2023,17 @@ module Prism
         # Visit a heredoc that can be either a string or an xstring.
         def visit_heredoc(node)
           children = Array.new
+          indented = false
+
+          # If this is a dedenting heredoc, then we need to insert the opening
+          # content into the children as well.
+          if node.opening.start_with?("<<~") && node.parts.length > 0 && !node.parts.first.is_a?(StringNode)
+            location = node.parts.first.location
+            location = location.copy(start_offset: location.start_offset - location.start_line_slice.bytesize)
+            children << builder.string_internal(token(location))
+            indented = true
+          end
+
           node.parts.each do |part|
             pushing =
               if part.is_a?(StringNode) && part.unescaped.include?("\n")
@@ -2080,8 +2073,10 @@ module Prism
 
           closing = node.closing
           closing_t = [closing.chomp, srange_offsets(node.closing_loc.start_offset, node.closing_loc.end_offset - (closing[/\s+$/]&.length || 0))]
+          composed = yield children, closing_t
 
-          [children, closing_t]
+          composed = composed.updated(nil, children[1..-1]) if indented
+          composed
         end
 
         # Visit a numeric node and account for the optional sign.


Test patch:
diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb
index a76f193f..d5787a0c 100644
--- a/test/prism/ruby/parser_test.rb
+++ b/test/prism/ruby/parser_test.rb
@@ -42,34 +42,23 @@ Parser::AST::Node.prepend(
 
 module Prism
   class ParserTest < TestCase
-    # These files are erroring because of the parser gem being wrong.
-    skip_incorrect = [
-      "embdoc_no_newline_at_end.txt"
-    ]
-
     # These files are either failing to parse or failing to translate, so we'll
     # skip them for now.
-    skip_all = skip_incorrect | [
+    skip_all = [
       "dash_heredocs.txt",
-      "dos_endings.txt",
       "heredocs_with_ignored_newlines.txt",
       "regex.txt",
       "regex_char_width.txt",
       "spanning_heredoc.txt",
       "spanning_heredoc_newlines.txt",
       "unescaping.txt",
-      "seattlerb/backticks_interpolation_line.txt",
       "seattlerb/block_decomp_anon_splat_arg.txt",
       "seattlerb/block_decomp_arg_splat_arg.txt",
       "seattlerb/block_decomp_arg_splat.txt",
       "seattlerb/block_decomp_splat.txt",
       "seattlerb/block_paren_splat.txt",
       "seattlerb/bug190.txt",
-      "seattlerb/case_in_hash_pat_rest_solo.txt",
-      "seattlerb/case_in_hash_pat_rest.txt",
-      "seattlerb/case_in.txt",
       "seattlerb/heredoc_nested.txt",
-      "seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt",
       "seattlerb/heredoc_with_carriage_return_escapes_windows.txt",
       "seattlerb/heredoc_with_carriage_return_escapes.txt",
       "seattlerb/heredoc_with_extra_carriage_returns_windows.txt",
@@ -77,24 +66,15 @@ module Prism
       "seattlerb/heredoc_with_only_carriage_returns.txt",
       "seattlerb/masgn_double_paren.txt",
       "seattlerb/parse_line_heredoc_hardnewline.txt",
-      "seattlerb/parse_pattern_044.txt",
-      "seattlerb/parse_pattern_058_2.txt",
-      "seattlerb/parse_pattern_058.txt",
-      "seattlerb/pct_nl.txt",
       "seattlerb/pctW_lineno.txt",
       "seattlerb/regexp_esc_C_slash.txt",
       "seattlerb/TestRubyParserShared.txt",
-      "unparser/corpus/literal/assignment.txt",
       "unparser/corpus/literal/block.txt",
-      "unparser/corpus/literal/def.txt",
-      "unparser/corpus/literal/dstr.txt",
       "unparser/corpus/literal/literal.txt",
       "unparser/corpus/literal/pattern.txt",
       "unparser/corpus/semantic/dstr.txt",
-      "unparser/corpus/semantic/opasgn.txt",
       "whitequark/dedenting_interpolating_heredoc_fake_line_continuation.txt",
       "whitequark/masgn_nested.txt",
-      "whitequark/newline_in_hash_argument.txt",
       "whitequark/parser_bug_640.txt",
       "whitequark/parser_slash_slash_n_escaping_in_literals.txt",
       "whitequark/ruby_bug_11989.txt",
@@ -111,13 +91,17 @@ module Prism
     # output expected by the parser gem, so we'll skip them for now.
     skip_tokens = [
       "comments.txt",
+      "dos_endings.txt",
+      "embdoc_no_newline_at_end.txt",
       "heredoc_with_comment.txt",
       "indented_file_end.txt",
       "methods.txt",
       "strings.txt",
       "tilde_heredocs.txt",
       "xstring_with_backslash.txt",
+      "seattlerb/backticks_interpolation_line.txt",
       "seattlerb/bug169.txt",
+      "seattlerb/case_in.txt",
       "seattlerb/class_comments.txt",
       "seattlerb/difficult4__leading_dots2.txt",
       "seattlerb/difficult6__7.txt",
@@ -126,6 +110,7 @@ module Prism
       "seattlerb/heredoc__backslash_dos_format.txt",
       "seattlerb/heredoc_backslash_nl.txt",
       "seattlerb/heredoc_comma_arg.txt",
+      "seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt",
       "seattlerb/heredoc_squiggly_blank_lines.txt",
       "seattlerb/heredoc_squiggly_interp.txt",
       "seattlerb/heredoc_squiggly_tabs_extra.txt",
@@ -164,6 +149,9 @@ module Prism
       "seattlerb/str_single_newline.txt",
       "seattlerb/symbol_empty.txt",
       "seattlerb/symbols_empty_space.txt",
+      "unparser/corpus/literal/assignment.txt",
+      "unparser/corpus/literal/dstr.txt",
+      "unparser/corpus/semantic/opasgn.txt",
       "whitequark/args.txt",
       "whitequark/beginless_erange_after_newline.txt",
       "whitequark/beginless_irange_after_newline.txt",
@@ -175,6 +163,7 @@ module Prism
       "whitequark/interp_digit_var.txt",
       "whitequark/lbrace_arg_after_command_args.txt",
       "whitequark/multiple_pattern_matches.txt",
+      "whitequark/newline_in_hash_argument.txt",
       "whitequark/parser_drops_truncated_parts_of_squiggly_heredoc.txt",
       "whitequark/ruby_bug_11990.txt",
       "whitequark/ruby_bug_14690.txt",
@@ -183,15 +172,20 @@ module Prism
       "whitequark/space_args_block.txt"
     ]
 
-    Fixture.each(except: skip_all) do |fixture|
+    Fixture.each do |fixture|
       define_method(fixture.test_name) do
-        assert_equal_parses(fixture, compare_tokens: !skip_tokens.include?(fixture.path))
+        assert_equal_parses(
+          fixture,
+          compare_asts: !skip_all.include?(fixture.path),
+          compare_tokens: !skip_tokens.include?(fixture.path),
+          compare_comments: fixture.path != "embdoc_no_newline_at_end.txt"
+        )
       end
     end
 
     private
 
-    def assert_equal_parses(fixture, compare_tokens: true)
+    def assert_equal_parses(fixture, compare_asts: true, compare_tokens: true, compare_comments: true)
       buffer = Parser::Source::Buffer.new(fixture.path, 1)
       buffer.source = fixture.read
 
@@ -209,9 +203,17 @@ module Prism
       actual_ast, actual_comments, actual_tokens =
         ignore_warnings { Prism::Translation::Parser33.new.tokenize(buffer) }
 
-      assert_equal expected_ast, actual_ast, -> { assert_equal_asts_message(expected_ast, actual_ast) }
-      assert_equal_tokens(expected_tokens, actual_tokens) if compare_tokens
-      assert_equal_comments(expected_comments, actual_comments)
+      if expected_ast == actual_ast
+        if !compare_asts
+          puts "#{fixture.path} is now passing"
+        end
+
+        assert_equal expected_ast, actual_ast, -> { assert_equal_asts_message(expected_ast, actual_ast) }
+        assert_equal_tokens(expected_tokens, actual_tokens) if compare_tokens
+        assert_equal_comments(expected_comments, actual_comments) if compare_comments
+      elsif compare_asts
+        assert_equal expected_ast, actual_ast, -> { assert_equal_asts_message(expected_ast, actual_ast) }
+      end
     end
 
     def assert_equal_asts_message(expected_ast, actual_ast)