Repository: ruby/prism
Pull request #2586 (2024-03-12 17:25:17)

Fix a token incompatibility for `Prism::Translation::Parser::Lexer`
In practice, the `BACKTICK` is mapped either as `:tXSTRING_BEG` or `:tBACK_REF2`. The former is used in xstrings like `` `foo` ``, while the latter is utilized as a back reference in contexts like `` A::` ``.
This PR will make corrections to differentiate the use of `BACKTICK`.

This mistake was discovered through the investigation of xstring.txt file. The PR will run tests from xstring.txt file except for `` `f\oo` ``, which will still fail, hence it will be separated into xstring_with_backslash.txt file. This separation will facilitate addressing the correction at a different time.


Patch:
diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb
index 054e56a4..1d9b0752 100644
--- a/lib/prism/translation/parser/lexer.rb
+++ b/lib/prism/translation/parser/lexer.rb
@@ -24,7 +24,7 @@ module Prism
           AMPERSAND_DOT: :tANDDOT,
           AMPERSAND_EQUAL: :tOP_ASGN,
           BACK_REFERENCE: :tBACK_REF,
-          BACKTICK: :tBACK_REF2,
+          BACKTICK: :tXSTRING_BEG,
           BANG: :tBANG,
           BANG_EQUAL: :tNEQ,
           BANG_TILDE: :tNMATCH,
@@ -250,7 +250,7 @@ module Prism
             when :tNL
               value = nil
             when :tFLOAT
-              value = parse_float(value)
+              value = Float(value)
             when :tIMAGINARY
               value = parse_complex(value)
             when :tINTEGER
@@ -259,7 +259,7 @@ module Prism
                 location = Range.new(source_buffer, offset_cache[token.location.start_offset + 1], offset_cache[token.location.end_offset])
               end
 
-              value = parse_integer(value)
+              value = Integer(value)
             when :tLABEL
               value.chomp!(":")
             when :tLABEL_END
@@ -267,7 +267,7 @@ module Prism
             when :tLCURLY
               type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
             when :tNTH_REF
-              value = parse_integer(value.delete_prefix("$"))
+              value = Integer(value.delete_prefix("$"))
             when :tOP_ASGN
               value.chomp!("=")
             when :tRATIONAL
@@ -325,6 +325,10 @@ module Prism
               if !tokens.empty? && tokens.dig(-1, 0) == :kDEF
                 type = :tIDENTIFIER
               end
+            when :tXSTRING_BEG
+              if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT
+                type = :tBACK_REF2
+              end
             end
 
             tokens << [type, [value, location]]
@@ -339,20 +343,6 @@ module Prism
 
         private
 
-        # Parse an integer from the string representation.
-        def parse_integer(value)
-          Integer(value)
-        rescue ArgumentError
-          0
-        end
-
-        # Parse a float from the string representation.
-        def parse_float(value)
-          Float(value)
-        rescue ArgumentError
-          0.0
-        end
-
         # Parse a complex from the string representation.
         def parse_complex(value)
           value.chomp!("i")
@@ -360,12 +350,10 @@ module Prism
           if value.end_with?("r")
             Complex(0, parse_rational(value))
           elsif value.start_with?(/0[BbOoDdXx]/)
-            Complex(0, parse_integer(value))
+            Complex(0, Integer(value))
           else
             Complex(0, value)
           end
-        rescue ArgumentError
-          0i
         end
 
         # Parse a rational from the string representation.
@@ -373,12 +361,10 @@ module Prism
           value.chomp!("r")
 
           if value.start_with?(/0[BbOoDdXx]/)
-            Rational(parse_integer(value))
+            Rational(Integer(value))
           else
             Rational(value)
           end
-        rescue ArgumentError
-          0r
         end
       end
     end


Test patch:
diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt
index 01bcc09f..623afd97 100644
--- a/test/prism/fixtures/xstring.txt
+++ b/test/prism/fixtures/xstring.txt
@@ -2,6 +2,4 @@
 
 `foo #{bar} baz`
 
-`f\oo`
-
 `foo`
diff --git a/test/prism/fixtures/xstring_with_backslash.txt b/test/prism/fixtures/xstring_with_backslash.txt
new file mode 100644
index 00000000..b51bb0f6
--- /dev/null
+++ b/test/prism/fixtures/xstring_with_backslash.txt
@@ -0,0 +1,1 @@
+`f\oo`
diff --git a/test/prism/parser_test.rb b/test/prism/parser_test.rb
index 60fe0e0f..012a2996 100644
--- a/test/prism/parser_test.rb
+++ b/test/prism/parser_test.rb
@@ -76,8 +76,9 @@ module Prism
       "heredocs_leading_whitespace.txt",
       "heredocs_nested.txt",
       "indented_file_end.txt",
+      "non_alphanumeric_methods.txt",
       "strings.txt",
-      "xstring.txt"
+      "xstring_with_backslash.txt"
     ]
 
     Dir["*.txt", base: base].each do |name|
diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt
index 325a39af..0a9d0758 100644
--- a/test/prism/snapshots/xstring.txt
+++ b/test/prism/snapshots/xstring.txt
@@ -1,8 +1,8 @@
-@ ProgramNode (location: (1,0)-(7,5))
+@ ProgramNode (location: (1,0)-(5,5))
 ├── locals: []
 └── statements:
-    @ StatementsNode (location: (1,0)-(7,5))
-    └── body: (length: 4)
+    @ StatementsNode (location: (1,0)-(5,5))
+    └── body: (length: 3)
         ├── @ XStringNode (location: (1,0)-(1,7))
         │   ├── flags: ∅
         │   ├── opening_loc: (1,0)-(1,3) = "%x["
@@ -41,15 +41,9 @@
         │   │       ├── closing_loc: ∅
         │   │       └── unescaped: " baz"
         │   └── closing_loc: (3,15)-(3,16) = "`"
-        ├── @ XStringNode (location: (5,0)-(5,6))
-        │   ├── flags: ∅
-        │   ├── opening_loc: (5,0)-(5,1) = "`"
-        │   ├── content_loc: (5,1)-(5,5) = "f\\oo"
-        │   ├── closing_loc: (5,5)-(5,6) = "`"
-        │   └── unescaped: "foo"
-        └── @ XStringNode (location: (7,0)-(7,5))
+        └── @ XStringNode (location: (5,0)-(5,5))
             ├── flags: ∅
-            ├── opening_loc: (7,0)-(7,1) = "`"
-            ├── content_loc: (7,1)-(7,4) = "foo"
-            ├── closing_loc: (7,4)-(7,5) = "`"
+            ├── opening_loc: (5,0)-(5,1) = "`"
+            ├── content_loc: (5,1)-(5,4) = "foo"
+            ├── closing_loc: (5,4)-(5,5) = "`"
             └── unescaped: "foo"
diff --git a/test/prism/snapshots/xstring_with_backslash.txt b/test/prism/snapshots/xstring_with_backslash.txt
new file mode 100644
index 00000000..7e0fa1ab
--- /dev/null
+++ b/test/prism/snapshots/xstring_with_backslash.txt
@@ -0,0 +1,11 @@
+@ ProgramNode (location: (1,0)-(1,6))
+├── locals: []
+└── statements:
+    @ StatementsNode (location: (1,0)-(1,6))
+    └── body: (length: 1)
+        └── @ XStringNode (location: (1,0)-(1,6))
+            ├── flags: ∅
+            ├── opening_loc: (1,0)-(1,1) = "`"
+            ├── content_loc: (1,1)-(1,5) = "f\\oo"
+            ├── closing_loc: (1,5)-(1,6) = "`"
+            └── unescaped: "foo"