Repository: ruby/prism
Pull request #2627 (2024-03-25 13:48:03)

CLRF in string-like structures
Fixes https://github.com/ruby/prism/issues/2585


Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 87023683..8368139a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -109,7 +109,7 @@ jobs:
         ruby-version: head
         bundler-cache: true
     - name: Run Ruby tests
-      run: bundle exec rake compile_no_debug && bundle exec rake test
+      run: bundle exec rake compile_no_debug test
 
   build-debug:
     strategy:
@@ -139,7 +139,7 @@ jobs:
         ruby-version: head
         bundler-cache: true
     - name: Run Ruby tests
-      run: bundle exec rake compile_minimal && bundle exec rake test
+      run: bundle exec rake compile_minimal test
       env:
         PRISM_BUILD_MINIMAL: "1"
 
diff --git a/Makefile b/Makefile
index d18a9b97..b97d1b34 100644
--- a/Makefile
+++ b/Makefile
@@ -94,7 +94,7 @@ all-no-debug: DEBUG_FLAGS := -DNDEBUG=1
 all-no-debug: OPTFLAGS := -O3
 all-no-debug: all
 
-minimal: CFLAGS := $(CFLAGS) -DPRISM_EXCLUDE_SERIALIZATION -DPRISM_EXCLUDE_JSON -DPRISM_EXCLUDE_PACK -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_ENCODING_EXCLUDE_FULL
+minimal: CFLAGS := $(CFLAGS) -DPRISM_BUILD_MINIMAL
 minimal: all
 
 run: Makefile $(STATIC_OBJECTS) $(HEADERS) test.c
diff --git a/Rakefile b/Rakefile
index 34a648f4..142bca2d 100644
--- a/Rakefile
+++ b/Rakefile
@@ -4,11 +4,7 @@ require "bundler/gem_tasks"
 require "rake/extensiontask"
 require "rake/clean"
 
-task compile: :make
-task compile_no_debug: :make_no_debug
-task compile_minimal: :make_minimal
-
-task default: [:compile, :test]
+task default: %i[compile test]
 
 require_relative "templates/template"
 
@@ -20,6 +16,10 @@ task(make: :templates) { sh(make) }
 task(make_no_debug: :templates) { sh("#{make} all-no-debug") }
 task(make_minimal: :templates) { sh("#{make} minimal") }
 
+task compile: :make
+task compile_no_debug: %i[make_no_debug compile]
+task compile_minimal: %i[make_minimal compile]
+
 # decorate the gem build task with prerequisites
 task build: [:check_manifest, :templates]
 
diff --git a/docs/build_system.md b/docs/build_system.md
index d2ada972..0b771d20 100644
--- a/docs/build_system.md
+++ b/docs/build_system.md
@@ -112,6 +112,7 @@ MAKEFLAGS="-j10" bundle exec rake compile
 ## Build options
 
 * `PRISM_BUILD_DEBUG` - Will cause all file reading to copy into its own allocation to allow easier tracking of reading off the end of the buffer. By default this is off.
+* `PRISM_BUILD_MINIMAL` - Define all of the `PRISM_EXCLUDE_*` flags at once.
 * `PRISM_ENCODING_EXCLUDE_FULL` - Will cause the library to exclude the full encoding API, and only include the minimal number of encodings to support parsing Ruby code without encoding comments. By default this is off.
 * `PRISM_EXPORT_SYMBOLS` - Will cause the shared library to export symbols. By default this is off.
 * `PRISM_EXCLUDE_JSON` - Will cause the library to exclude the JSON API. By default this is off.
diff --git a/ext/prism/api_pack.c b/ext/prism/api_pack.c
index c9f0b18a..98509ae6 100644
--- a/ext/prism/api_pack.c
+++ b/ext/prism/api_pack.c
@@ -1,5 +1,12 @@
 #include "prism/extension.h"
 
+#ifdef PRISM_EXCLUDE_PACK
+
+void
+Init_prism_pack(void) {}
+
+#else
+
 static VALUE rb_cPrism;
 static VALUE rb_cPrismPack;
 static VALUE rb_cPrismPackDirective;
@@ -265,3 +272,5 @@ Init_prism_pack(void) {
     pack_symbol = ID2SYM(rb_intern("pack"));
     unpack_symbol = ID2SYM(rb_intern("unpack"));
 }
+
+#endif
diff --git a/ext/prism/extconf.rb b/ext/prism/extconf.rb
index 86bbe90e..32b08bb6 100644
--- a/ext/prism/extconf.rb
+++ b/ext/prism/extconf.rb
@@ -6,9 +6,13 @@ if ARGV.delete("--help")
 
       Flags that are always valid:
 
-          --enable-debug-mode-build
-              Enable debug mode build.
-              You may also use set PRISM_BUILD_DEBUG environment variable.
+          --enable-build-debug
+              Enable debug build.
+              You may also set the PRISM_BUILD_DEBUG environment variable.
+
+          --enable-build-minimal
+              Enable minimal build.
+              You may also set the PRISM_BUILD_MINIMAL environment variable.
 
           --help
               Display this message.
@@ -16,7 +20,10 @@ if ARGV.delete("--help")
       Environment variables used:
 
           PRISM_BUILD_DEBUG
-              Equivalent to `--enable-debug-mode-build` when set, even if nil or blank.
+              Equivalent to `--enable-build-debug` when set, even if nil or blank.
+
+          PRISM_BUILD_MINIMAL
+              Equivalent to `--enable-build-minimal` when set, even if nil or blank.
 
   TEXT
   exit!(0)
@@ -71,15 +78,24 @@ unless find_header("prism/extension.h", File.expand_path("..", __dir__))
   raise "prism/extension.h is required"
 end
 
-# If `--enable-debug-mode-build` is passed to this script or the
+# If `--enable-build-debug` is passed to this script or the
 # `PRISM_BUILD_DEBUG` environment variable is defined, we'll build with the
 # `PRISM_BUILD_DEBUG` macro defined. This causes parse functions to
 # duplicate their input so that they have clearly set bounds, which is useful
 # for finding bugs that cause the parser to read off the end of the input.
-if enable_config("debug-mode-build", ENV["PRISM_BUILD_DEBUG"] || false)
+if enable_config("build-debug", ENV["PRISM_BUILD_DEBUG"] || false)
   append_cflags("-DPRISM_BUILD_DEBUG")
 end
 
+# If `--enable-build-minimal` is passed to this script or the
+# `PRISM_BUILD_MINIMAL` environment variable is defined, we'll build with the
+# set of defines that comprise the minimal set. This causes the parser to be
+# built with minimal features, necessary for stripping out functionality when
+# the size of the final built artifact is a concern.
+if enable_config("build-minimal", ENV["PRISM_BUILD_MINIMAL"] || false)
+  append_cflags("-DPRISM_BUILD_MINIMAL")
+end
+
 # By default, all symbols are hidden in the shared library.
 append_cflags("-fvisibility=hidden")
 
diff --git a/ext/prism/extension.c b/ext/prism/extension.c
index 7c8636e3..88ba006a 100644
--- a/ext/prism/extension.c
+++ b/ext/prism/extension.c
@@ -270,6 +270,8 @@ file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) {
     }
 }
 
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+
 /******************************************************************************/
 /* Serializing the AST                                                        */
 /******************************************************************************/
@@ -351,6 +353,8 @@ dump_file(int argc, VALUE *argv, VALUE self) {
     return value;
 }
 
+#endif
+
 /******************************************************************************/
 /* Extracting values for the parse result                                     */
 /******************************************************************************/
@@ -1129,6 +1133,8 @@ profile_file(VALUE self, VALUE filepath) {
     return Qnil;
 }
 
+#ifndef PRISM_EXCLUDE_PRETTYPRINT
+
 /**
  * call-seq:
  *   Debug::inspect_node(source) -> inspected
@@ -1159,6 +1165,8 @@ inspect_node(VALUE self, VALUE source) {
     return string;
 }
 
+#endif
+
 /**
  * call-seq:
  *   Debug::format_errors(source, colorize) -> String
@@ -1349,8 +1357,6 @@ Init_prism(void) {
     rb_define_const(rb_cPrism, "VERSION", rb_str_new2(EXPECTED_PRISM_VERSION));
 
     // First, the functions that have to do with lexing and parsing.
-    rb_define_singleton_method(rb_cPrism, "dump", dump, -1);
-    rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1);
     rb_define_singleton_method(rb_cPrism, "lex", lex, -1);
     rb_define_singleton_method(rb_cPrism, "lex_file", lex_file, -1);
     rb_define_singleton_method(rb_cPrism, "parse", parse, -1);
@@ -1363,6 +1369,11 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrism, "parse_success?", parse_success_p, -1);
     rb_define_singleton_method(rb_cPrism, "parse_file_success?", parse_file_success_p, -1);
 
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+    rb_define_singleton_method(rb_cPrism, "dump", dump, -1);
+    rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1);
+#endif
+
     // Next, the functions that will be called by the parser to perform various
     // internal tasks. We expose these to make them easier to test.
     VALUE rb_cPrismDebug = rb_define_module_under(rb_cPrism, "Debug");
@@ -1370,10 +1381,13 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrismDebug, "integer_parse", integer_parse, 1);
     rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1);
     rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
     rb_define_singleton_method(rb_cPrismDebug, "format_errors", format_errors, 2);
     rb_define_singleton_method(rb_cPrismDebug, "static_inspect", static_inspect, -1);
 
+#ifndef PRISM_EXCLUDE_PRETTYPRINT
+    rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
+#endif
+
     // Next, define the functions that are exposed through the private
     // Debug::Encoding class.
     rb_cPrismDebugEncoding = rb_define_class_under(rb_cPrismDebug, "Encoding", rb_cObject);
diff --git a/include/prism/defines.h b/include/prism/defines.h
index aca3c6dc..2fe73fe3 100644
--- a/include/prism/defines.h
+++ b/include/prism/defines.h
@@ -182,4 +182,25 @@
     #endif
 #endif
 
+/**
+ * If PRISM_BUILD_MINIMAL is defined, then we're going to define every possible
+ * switch that will turn off certain features of prism.
+ */
+#ifdef PRISM_BUILD_MINIMAL
+    /** Exclude the serialization API. */
+    #define PRISM_EXCLUDE_SERIALIZATION
+
+    /** Exclude the JSON serialization API. */
+    #define PRISM_EXCLUDE_JSON
+
+    /** Exclude the Array#pack parser API. */
+    #define PRISM_EXCLUDE_PACK
+
+    /** Exclude the prettyprint API. */
+    #define PRISM_EXCLUDE_PRETTYPRINT
+
+    /** Exclude the full set of encodings, using the minimal only. */
+    #define PRISM_ENCODING_EXCLUDE_FULL
+#endif
+
 #endif
diff --git a/include/prism/pack.h b/include/prism/pack.h
index cfdc251f..0b0b4b19 100644
--- a/include/prism/pack.h
+++ b/include/prism/pack.h
@@ -6,6 +6,8 @@
 #ifndef PRISM_PACK_H
 #define PRISM_PACK_H
 
+#include "prism/defines.h"
+
 // We optionally support parsing String#pack templates. For systems that don't
 // want or need this functionality, it can be turned off with the
 // PRISM_EXCLUDE_PACK define.
@@ -15,8 +17,6 @@ void pm_pack_parse(void);
 
 #else
 
-#include "prism/defines.h"
-
 #include <stdint.h>
 #include <stdlib.h>
 
diff --git a/include/prism/parser.h b/include/prism/parser.h
index b685fa37..f706a67d 100644
--- a/include/prism/parser.h
+++ b/include/prism/parser.h
@@ -6,8 +6,8 @@
 #ifndef PRISM_PARSER_H
 #define PRISM_PARSER_H
 
-#include "prism/ast.h"
 #include "prism/defines.h"
+#include "prism/ast.h"
 #include "prism/encoding.h"
 #include "prism/options.h"
 #include "prism/util/pm_constant_pool.h"
@@ -173,7 +173,7 @@ typedef struct pm_lex_mode {
              * This is the character set that should be used to delimit the
              * tokens within the regular expression.
              */
-            uint8_t breakpoints[6];
+            uint8_t breakpoints[7];
         } regexp;
 
         struct {
@@ -206,7 +206,7 @@ typedef struct pm_lex_mode {
              * This is the character set that should be used to delimit the
              * tokens within the string.
              */
-            uint8_t breakpoints[6];
+            uint8_t breakpoints[7];
         } string;
 
         struct {
diff --git a/include/prism/prettyprint.h b/include/prism/prettyprint.h
index ea11b4a2..5a52b2b6 100644
--- a/include/prism/prettyprint.h
+++ b/include/prism/prettyprint.h
@@ -6,14 +6,14 @@
 #ifndef PRISM_PRETTYPRINT_H
 #define PRISM_PRETTYPRINT_H
 
+#include "prism/defines.h"
+
 #ifdef PRISM_EXCLUDE_PRETTYPRINT
 
 void pm_prettyprint(void);
 
 #else
 
-#include "prism/defines.h"
-
 #include <stdio.h>
 
 #include "prism/ast.h"
diff --git a/src/prism.c b/src/prism.c
index 936f01b4..74bd8b62 100644
--- a/src/prism.c
+++ b/src/prism.c
@@ -308,14 +308,14 @@ lex_mode_push_regexp(pm_parser_t *parser, uint8_t incrementor, uint8_t terminato
     // regular expression. We'll use strpbrk to find the first of these
     // characters.
     uint8_t *breakpoints = lex_mode.as.regexp.breakpoints;
-    memcpy(breakpoints, "\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints));
+    memcpy(breakpoints, "\r\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints));
 
     // First we'll add the terminator.
-    breakpoints[3] = terminator;
+    breakpoints[4] = terminator;
 
     // Next, if there is an incrementor, then we'll check for that as well.
     if (incrementor != '\0') {
-        breakpoints[4] = incrementor;
+        breakpoints[5] = incrementor;
     }
 
     return lex_mode_push(parser, lex_mode);
@@ -340,10 +340,10 @@ lex_mode_push_string(pm_parser_t *parser, bool interpolation, bool label_allowed
     // These are the places where we need to split up the content of the
     // string. We'll use strpbrk to find the first of these characters.
     uint8_t *breakpoints = lex_mode.as.string.breakpoints;
-    memcpy(breakpoints, "\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints));
+    memcpy(breakpoints, "\r\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints));
 
     // Now add in the terminator.
-    size_t index = 2;
+    size_t index = 3;
     breakpoints[index++] = terminator;
 
     // If interpolation is allowed, then we're going to check for the #
@@ -1238,7 +1238,7 @@ pm_regular_expression_flags_create(pm_parser_t *parser, const pm_token_t *closin
 
         size_t unknown_flags_length = pm_buffer_length(&unknown_flags);
         if (unknown_flags_length != 0) {
-            const char *word = unknown_flags_length >= 2 ? "options" : "option";
+            char *word = unknown_flags_length >= 2 ? "options" : "option";
             PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_REGEXP_UNKNOWN_OPTIONS, word, unknown_flags_length, pm_buffer_value(&unknown_flags));
         }
         pm_buffer_free(&unknown_flags);
@@ -10783,36 +10783,6 @@ parser_lex(pm_parser_t *parser) {
             pm_regexp_token_buffer_t token_buffer = { 0 };
 
             while (breakpoint != NULL) {
-                // If we hit a null byte, skip directly past it.
-                if (*breakpoint == '\0') {
-                    parser->current.end = breakpoint + 1;
-                    breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
-                    continue;
-                }
-
-                // If we've hit a newline, then we need to track that in the
-                // list of newlines.
-                if (*breakpoint == '\n') {
-                    // For the special case of a newline-terminated regular expression, we will pass
-                    // through this branch twice -- once with PM_TOKEN_REGEXP_BEGIN and then again
-                    // with PM_TOKEN_STRING_CONTENT. Let's avoid tracking the newline twice, by
-                    // tracking it only in the REGEXP_BEGIN case.
-                    if (
-                        !(lex_mode->as.regexp.terminator == '\n' && parser->current.type != PM_TOKEN_REGEXP_BEGIN)
-                        && parser->heredoc_end == NULL
-                    ) {
-                        pm_newline_list_append(&parser->newline_list, breakpoint);
-                    }
-
-                    if (lex_mode->as.regexp.terminator != '\n') {
-                        // If the terminator is not a newline, then we can set
-                        // the next breakpoint and continue.
-                        parser->current.end = breakpoint + 1;
-                        breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
-                        continue;
-                    }
-                }
-
                 // If we hit the terminator, we need to determine what kind of
                 // token to return.
                 if (*breakpoint == lex_mode->as.regexp.terminator) {
@@ -10832,9 +10802,17 @@ parser_lex(pm_parser_t *parser) {
                         LEX(PM_TOKEN_STRING_CONTENT);
                     }
 
+                    // Check here if we need to track the newline.
+                    size_t eol_length = match_eol_at(parser, breakpoint);
+                    if (eol_length) {
+                        parser->current.end = breakpoint + eol_length;
+                        pm_newline_list_append(&parser->newline_list, parser->current.end - 1);
+                    } else {
+                        parser->current.end = breakpoint + 1;
+                    }
+
                     // Since we've hit the terminator of the regular expression,
                     // we now need to parse the options.
-                    parser->current.end = breakpoint + 1;
                     parser->current.end += pm_strspn_regexp_option(parser->current.end, parser->end - parser->current.end);
 
                     lex_mode_pop(parser);
@@ -10842,114 +10820,148 @@ parser_lex(pm_parser_t *parser) {
                     LEX(PM_TOKEN_REGEXP_END);
                 }
 
-                // If we hit escapes, then we need to treat the next token
-                // literally. In this case we'll skip past the next character
+                // If we've hit the incrementor, then we need to skip past it
                 // and find the next breakpoint.
-                if (*breakpoint == '\\') {
+                if (*breakpoint && *breakpoint == lex_mode->as.regexp.incrementor) {
                     parser->current.end = breakpoint + 1;
+                    breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
+                    lex_mode->as.regexp.nesting++;
+                    continue;
+                }
 
-                    // If we've hit the end of the file, then break out of the
-                    // loop by setting the breakpoint to NULL.
-                    if (parser->current.end == parser->end) {
-                        breakpoint = NULL;
-                        continue;
-                    }
+                switch (*breakpoint) {
+                    case '\0':
+                        // If we hit a null byte, skip directly past it.
+                        parser->current.end = breakpoint + 1;
+                        breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
+                        break;
+                    case '\r':
+                        if (peek_at(parser, breakpoint + 1) != '\n') {
+                            parser->current.end = breakpoint + 1;
+                            breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
+                            break;
+                        }
 
-                    pm_regexp_token_buffer_escape(parser, &token_buffer);
-                    uint8_t peeked = peek(parser);
+                        breakpoint++;
+                        parser->current.end = breakpoint;
+                        pm_regexp_token_buffer_escape(parser, &token_buffer);
+                        token_buffer.base.cursor = breakpoint;
 
-                    switch (peeked) {
-                        case '\r':
-                            parser->current.end++;
-                            if (peek(parser) != '\n') {
-                                if (lex_mode->as.regexp.terminator != '\r') {
-                                    pm_token_buffer_push_byte(&token_buffer.base, '\\');
-                                }
-                                pm_regexp_token_buffer_push_byte(&token_buffer, '\r');
-                                pm_token_buffer_push_byte(&token_buffer.base, '\r');
-                                break;
-                            }
                         /* fallthrough */
-                        case '\n':
-                            if (parser->heredoc_end) {
-                                // ... if we are on the same line as a heredoc,
-                                // flush the heredoc and continue parsing after
-                                // heredoc_end.
-                                parser_flush_heredoc_end(parser);
-                                pm_regexp_token_buffer_copy(parser, &token_buffer);
-                                LEX(PM_TOKEN_STRING_CONTENT);
-                            } else {
-                                // ... else track the newline.
-                                pm_newline_list_append(&parser->newline_list, parser->current.end);
-                            }
+                    case '\n':
+                        // If we've hit a newline, then we need to track that in
+                        // the list of newlines.
+                        if (parser->heredoc_end == NULL) {
+                            pm_newline_list_append(&parser->newline_list, breakpoint);
+                        }
 
-                            parser->current.end++;
-                            break;
-                        case 'c':
-                        case 'C':
-                        case 'M':
-                        case 'u':
-                        case 'x':
-                            escape_read(parser, &token_buffer.regexp_buffer, &token_buffer.base.buffer, PM_ESCAPE_FLAG_REGEXP);
+                        parser->current.end = breakpoint + 1;
+                        breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
+                        break;
+                    case '\\': {
+                        // If we hit escapes, then we need to treat the next
+                        // token literally. In this case we'll skip past the
+                        // next character and find the next breakpoint.
+                        parser->current.end = breakpoint + 1;
+
+                        // If we've hit the end of the file, then break out of
+                        // the loop by setting the breakpoint to NULL.
+                        if (parser->current.end == parser->end) {
+                            breakpoint = NULL;
                             break;
-                        default:
-                            if (lex_mode->as.regexp.terminator == peeked) {
-                                // Some characters when they are used as the
-                                // terminator also receive an escape. They are
-                                // enumerated here.
-                                switch (peeked) {
-                                    case '$': case ')': case '*': case '+':
-                                    case '.': case '>': case '?': case ']':
-                                    case '^': case '|': case '}':
+                        }
+
+                        pm_regexp_token_buffer_escape(parser, &token_buffer);
+                        uint8_t peeked = peek(parser);
+
+                        switch (peeked) {
+                            case '\r':
+                                parser->current.end++;
+                                if (peek(parser) != '\n') {
+                                    if (lex_mode->as.regexp.terminator != '\r') {
                                         pm_token_buffer_push_byte(&token_buffer.base, '\\');
-                                        break;
-                                    default:
-                                        break;
+                                    }
+                                    pm_regexp_token_buffer_push_byte(&token_buffer, '\r');
+                                    pm_token_buffer_push_byte(&token_buffer.base, '\r');
+                                    break;
+                                }
+                            /* fallthrough */
+                            case '\n':
+                                if (parser->heredoc_end) {
+                                    // ... if we are on the same line as a heredoc,
+                                    // flush the heredoc and continue parsing after
+                                    // heredoc_end.
+                                    parser_flush_heredoc_end(parser);
+                                    pm_regexp_token_buffer_copy(parser, &token_buffer);
+                                    LEX(PM_TOKEN_STRING_CONTENT);
+                                } else {
+                                    // ... else track the newline.
+                                    pm_newline_list_append(&parser->newline_list, parser->current.end);
                                 }
 
-                                pm_regexp_token_buffer_push_byte(&token_buffer, peeked);
-                                pm_token_buffer_push_byte(&token_buffer.base, peeked);
                                 parser->current.end++;
                                 break;
-                            }
-
-                            if (peeked < 0x80) pm_token_buffer_push_byte(&token_buffer.base, '\\');
-                            pm_regexp_token_buffer_push_escaped(&token_buffer, parser);
-                            break;
-                    }
+                            case 'c':
+                            case 'C':
+                            case 'M':
+                            case 'u':
+                            case 'x':
+                                escape_read(parser, &token_buffer.regexp_buffer, &token_buffer.base.buffer, PM_ESCAPE_FLAG_REGEXP);
+                                break;
+                            default:
+                                if (lex_mode->as.regexp.terminator == peeked) {
+                                    // Some characters when they are used as the
+                                    // terminator also receive an escape. They are
+                                    // enumerated here.
+                                    switch (peeked) {
+                                        case '$': case ')': case '*': case '+':
+                                        case '.': case '>': case '?': case ']':
+                                        case '^': case '|': case '}':
+                                            pm_token_buffer_push_byte(&token_buffer.base, '\\');
+                                            break;
+                                        default:
+                                            break;
+                                    }
 
-                    token_buffer.base.cursor = parser->current.end;
-                    breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
-                    continue;
-                }
+                                    pm_regexp_token_buffer_push_byte(&token_buffer, peeked);
+                                    pm_token_buffer_push_byte(&token_buffer.base, peeked);
+                                    parser->current.end++;
+                                    break;
+                                }
 
-                // If we hit a #, then we will attempt to lex interpolation.
-                if (*breakpoint == '#') {
-                    pm_token_type_t type = lex_interpolation(parser, breakpoint);
+                                if (peeked < 0x80) pm_token_buffer_push_byte(&token_buffer.base, '\\');
+                                pm_regexp_token_buffer_push_escaped(&token_buffer, parser);
+                                break;
+                        }
 
-                    if (type == PM_TOKEN_NOT_PROVIDED) {
-                        // If we haven't returned at this point then we had
-                        // something that looked like an interpolated class or
-                        // instance variable like "#@" but wasn't actually. In
-                        // this case we'll just skip to the next breakpoint.
+                        token_buffer.base.cursor = parser->current.end;
                         breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
-                        continue;
+                        break;
                     }
+                    case '#': {
+                        // If we hit a #, then we will attempt to lex
+                        // interpolation.
+                        pm_token_type_t type = lex_interpolation(parser, breakpoint);
 
-                    if (type == PM_TOKEN_STRING_CONTENT) {
-                        pm_regexp_token_buffer_flush(parser, &token_buffer);
-                    }
+                        if (type == PM_TOKEN_NOT_PROVIDED) {
+                            // If we haven't returned at this point then we had
+                            // something that looked like an interpolated class or
+                            // instance variable like "#@" but wasn't actually. In
+                            // this case we'll just skip to the next breakpoint.
+                            breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
+                            break;
+                        }
 
-                    LEX(type);
-                }
+                        if (type == PM_TOKEN_STRING_CONTENT) {
+                            pm_regexp_token_buffer_flush(parser, &token_buffer);
+                        }
 
-                // If we've hit the incrementor, then we need to skip past it
-                // and find the next breakpoint.
-                assert(*breakpoint == lex_mode->as.regexp.incrementor);
-                parser->current.end = breakpoint + 1;
-                breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false);
-                lex_mode->as.regexp.nesting++;
-                continue;
+                        LEX(type);
+                    }
+                    default:
+                        assert(false && "unreachable");
+                        break;
+                }
             }
 
             if (parser->current.end > parser->current.start) {
@@ -11042,29 +11054,43 @@ parser_lex(pm_parser_t *parser) {
                     LEX(PM_TOKEN_STRING_END);
                 }
 
-                // When we hit a newline, we need to flush any potential heredocs. Note
-                // that this has to happen after we check for the terminator in case the
-                // terminator is a newline character.
-                if (*breakpoint == '\n') {
-                    if (parser->heredoc_end == NULL) {
-                        pm_newline_list_append(&parser->newline_list, breakpoint);
-                        parser->current.end = breakpoint + 1;
-                        breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
-                        continue;
-                    } else {
-                        parser->current.end = breakpoint + 1;
-                        parser_flush_heredoc_end(parser);
-                        pm_token_buffer_flush(parser, &token_buffer);
-                        LEX(PM_TOKEN_STRING_CONTENT);
-                    }
-                }
-
                 switch (*breakpoint) {
                     case '\0':
                         // Skip directly past the null character.
                         parser->current.end = breakpoint + 1;
                         breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
                         break;
+                    case '\r':
+                        if (peek_at(parser, breakpoint + 1) != '\n') {
+                            parser->current.end = breakpoint + 1;
+                            breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
+                            break;
+                        }
+
+                        // If we hit a \r\n sequence, then we need to treat it
+                        // as a newline.
+                        breakpoint++;
+                        parser->current.end = breakpoint;
+                        pm_token_buffer_escape(parser, &token_buffer);
+                        token_buffer.cursor = breakpoint;
+
+                        /* fallthrough */
+                    case '\n':
+                        // When we hit a newline, we need to flush any potential
+                        // heredocs. Note that this has to happen after we check
+                        // for the terminator in case the terminator is a
+                        // newline character.
+                        if (parser->heredoc_end == NULL) {
+                            pm_newline_list_append(&parser->newline_list, breakpoint);
+                            parser->current.end = breakpoint + 1;
+                            breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
+                            break;
+                        }
+
+                        parser->current.end = breakpoint + 1;
+                        parser_flush_heredoc_end(parser);
+                        pm_token_buffer_flush(parser, &token_buffer);
+                        LEX(PM_TOKEN_STRING_CONTENT);
                     case '\\': {
                         // Here we hit escapes.
                         parser->current.end = breakpoint + 1;
@@ -11267,11 +11293,11 @@ parser_lex(pm_parser_t *parser) {
             // Otherwise we'll be parsing string content. These are the places
             // where we need to split up the content of the heredoc. We'll use
             // strpbrk to find the first of these characters.
-            uint8_t breakpoints[] = "\n\\#";
+            uint8_t breakpoints[] = "\r\n\\#";
 
             pm_heredoc_quote_t quote = lex_mode->as.heredoc.quote;
             if (quote == PM_HEREDOC_QUOTE_SINGLE) {
-                breakpoints[2] = '\0';
+                breakpoints[3] = '\0';
             }
 
             const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
@@ -11285,6 +11311,21 @@ parser_lex(pm_parser_t *parser) {
                         parser->current.end = breakpoint + 1;
                         breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
                         break;
+                    case '\r':
+                        parser->current.end = breakpoint + 1;
+
+                        if (peek_at(parser, breakpoint + 1) != '\n') {
+                            breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true);
+                            break;
+                        }
+
+                        // If we hit a \r\n sequence, then we want to replace it
+                        // with a single \n character in the final string.
+                        breakpoint++;
+                        pm_token_buffer_escape(parser, &token_buffer);
+                        token_buffer.cursor = breakpoint;
+
+                        /* fallthrough */
                     case '\n': {
                         if (parser->heredoc_end != NULL && (parser->heredoc_end > breakpoint)) {
                             parser_flush_heredoc_end(parser);


Test patch:
diff --git a/test/prism/encoding_test.rb b/test/prism/encoding_test.rb
index 649d05b8..2aee473d 100644
--- a/test/prism/encoding_test.rb
+++ b/test/prism/encoding_test.rb
@@ -344,7 +344,7 @@ module Prism
           next if ["/", "{"].include?(character)
 
           source = "# encoding: #{name}\n/(?##{character})/\n"
-          assert Prism.parse(source).success?
+          assert Prism.parse(source).success?, "Expected #{source.inspect} to parse successfully."
         end
       rescue RangeError
         source = "# encoding: #{name}\n\\x#{codepoint.to_s(16)}"
diff --git a/test/prism/ruby_parser_test.rb b/test/prism/ruby_parser_test.rb
index 952e493a..c9883574 100644
--- a/test/prism/ruby_parser_test.rb
+++ b/test/prism/ruby_parser_test.rb
@@ -52,25 +52,10 @@ module Prism
       whitequark/string_concat.txt
     ]
 
-    # These files contain CRLF line endings, which ruby_parser translates into
-    # LF before it gets back to the node. This means the node actually has the
-    # wrong contents.
-    crlf = %w[
-      dos_endings.txt
-      heredoc_with_comment.txt
-      seattlerb/heredoc__backslash_dos_format.txt
-      seattlerb/heredoc_with_carriage_return_escapes_windows.txt
-      seattlerb/heredoc_with_extra_carriage_horrible_mix.txt
-      seattlerb/heredoc_with_extra_carriage_returns_windows.txt
-      seattlerb/heredoc_with_extra_carriage_returns.txt
-      seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt
-      seattlerb/heredoc_with_only_carriage_returns_windows.txt
-      seattlerb/heredoc_with_only_carriage_returns.txt
-    ]
-
     # https://github.com/seattlerb/ruby_parser/issues/344
-    failures = crlf | %w[
+    failures = %w[
       alias.txt
+      dos_endings.txt
       heredocs_with_ignored_newlines.txt
       method_calls.txt
       methods.txt
@@ -79,8 +64,13 @@ module Prism
       patterns.txt
       regex.txt
       seattlerb/and_multi.txt
+      seattlerb/heredoc__backslash_dos_format.txt
       seattlerb/heredoc_bad_hex_escape.txt
       seattlerb/heredoc_bad_oct_escape.txt
+      seattlerb/heredoc_with_extra_carriage_horrible_mix.txt
+      seattlerb/heredoc_with_extra_carriage_returns_windows.txt
+      seattlerb/heredoc_with_only_carriage_returns_windows.txt
+      seattlerb/heredoc_with_only_carriage_returns.txt
       spanning_heredoc_newlines.txt
       spanning_heredoc.txt
       tilde_heredocs.txt
diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt
index ed75b8a5..c5b962f2 100644
--- a/test/prism/snapshots/dos_endings.txt
+++ b/test/prism/snapshots/dos_endings.txt
@@ -48,7 +48,7 @@
         │   ├── opening_loc: (7,0)-(7,4) = "<<-E"
         │   ├── content_loc: (8,0)-(11,0) = "    1 \\\r\n    2\r\n    3\r\n"
         │   ├── closing_loc: (11,0)-(12,0) = "E\r\n"
-        │   └── unescaped: "    1     2\r\n    3\r\n"
+        │   └── unescaped: "    1     2\n    3\n"
         ├── @ LocalVariableWriteNode (location: (13,0)-(15,0))
         │   ├── name: :x
         │   ├── depth: 0
@@ -94,7 +94,7 @@
             │   │           │   │       ├── opening_loc: ∅
             │   │           │   │       ├── content_loc: (19,0)-(20,0) = "    baz\r\n"
             │   │           │   │       ├── closing_loc: ∅
-            │   │           │   │       └── unescaped: "baz\r\n"
+            │   │           │   │       └── unescaped: "baz\n"
             │   │           │   └── closing_loc: (20,0)-(21,0) = "  EOF\r\n"
             │   │           ├── call_operator_loc: (17,14)-(17,15) = "."
             │   │           ├── name: :chop
diff --git a/test/prism/snapshots/heredoc_with_comment.txt b/test/prism/snapshots/heredoc_with_comment.txt
index 117fdc11..f2225ca9 100644
--- a/test/prism/snapshots/heredoc_with_comment.txt
+++ b/test/prism/snapshots/heredoc_with_comment.txt
@@ -11,7 +11,7 @@
             │   ├── opening_loc: (1,0)-(1,9) = "<<-TARGET"
             │   ├── content_loc: (2,0)-(3,0) = "  content makes for an obvious error\r\n"
             │   ├── closing_loc: (3,0)-(3,6) = "TARGET"
-            │   └── unescaped: "  content makes for an obvious error\r\n"
+            │   └── unescaped: "  content makes for an obvious error\n"
             ├── call_operator_loc: (1,9)-(1,10) = "."
             ├── name: :chomp
             ├── message_loc: (1,10)-(1,15) = "chomp"
diff --git a/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt b/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt
index 6ba437e3..353e4c69 100644
--- a/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt
+++ b/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt
@@ -13,5 +13,5 @@
             │   ├── opening_loc: (1,6)-(1,12) = "<<-XXX"
             │   ├── content_loc: (2,0)-(4,0) = "before\\\r\nafter\r\n"
             │   ├── closing_loc: (4,0)-(5,0) = "XXX\r\n"
-            │   └── unescaped: "beforeafter\r\n"
+            │   └── unescaped: "beforeafter\n"
             └── operator_loc: (1,4)-(1,5) = "="
diff --git a/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt b/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt
index 21802c57..2ef67633 100644
--- a/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt
@@ -8,4 +8,4 @@
             ├── opening_loc: (1,0)-(1,5) = "<<EOS"
             ├── content_loc: (2,0)-(4,0) = "foo\\rbar\r\nbaz\\r\r\n"
             ├── closing_loc: (4,0)-(5,0) = "EOS\r\n"
-            └── unescaped: "foo\rbar\r\nbaz\r\r\n"
+            └── unescaped: "foo\rbar\nbaz\r\n"
diff --git a/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_horrible_mix.txt b/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_horrible_mix.txt
index dca03e71..fbee0301 100644
--- a/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_horrible_mix.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_horrible_mix.txt
@@ -8,4 +8,4 @@
             ├── opening_loc: (1,0)-(1,7) = "<<'eot'"
             ├── content_loc: (2,0)-(3,0) = "body\r\n"
             ├── closing_loc: (3,0)-(4,0) = "eot\n"
-            └── unescaped: "body\r\n"
+            └── unescaped: "body\n"
diff --git a/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_returns.txt b/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_returns.txt
index 93fa1a16..b59203bc 100644
--- a/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_returns.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_returns.txt
@@ -8,4 +8,4 @@
             ├── opening_loc: (1,0)-(1,5) = "<<EOS"
             ├── content_loc: (2,0)-(4,0) = "foo\rbar\r\nbaz\n"
             ├── closing_loc: (4,0)-(5,0) = "EOS\n"
-            └── unescaped: "foo\rbar\r\nbaz\n"
+            └── unescaped: "foo\rbar\nbaz\n"
diff --git a/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_returns_windows.txt b/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_returns_windows.txt
index 98b6e3fe..36bc4c65 100644
--- a/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_returns_windows.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_with_extra_carriage_returns_windows.txt
@@ -8,4 +8,4 @@
             ├── opening_loc: (1,0)-(1,5) = "<<EOS"
             ├── content_loc: (2,0)-(4,0) = "foo\rbar\r\r\nbaz\r\n"
             ├── closing_loc: (4,0)-(5,0) = "EOS\r\n"
-            └── unescaped: "foo\rbar\r\r\nbaz\r\n"
+            └── unescaped: "foo\rbar\r\nbaz\n"
diff --git a/test/prism/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt b/test/prism/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt
index dc8f8ae6..7eb04bdb 100644
--- a/test/prism/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt
@@ -22,5 +22,5 @@
             │       ├── opening_loc: ∅
             │       ├── content_loc: (2,10)-(3,0) = "\r\n"
             │       ├── closing_loc: ∅
-            │       └── unescaped: "\r\n"
+            │       └── unescaped: "\n"
             └── closing_loc: (3,0)-(4,0) = "EOS\r\n"
diff --git a/test/prism/snapshots/seattlerb/heredoc_with_only_carriage_returns.txt b/test/prism/snapshots/seattlerb/heredoc_with_only_carriage_returns.txt
index 6771f4af..6a535c64 100644
--- a/test/prism/snapshots/seattlerb/heredoc_with_only_carriage_returns.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_with_only_carriage_returns.txt
@@ -8,4 +8,4 @@
             ├── opening_loc: (1,0)-(1,5) = "<<EOS"
             ├── content_loc: (2,0)-(5,0) = "\r\n\r\r\n\\r\n"
             ├── closing_loc: (5,0)-(6,0) = "EOS\n"
-            └── unescaped: "\r\n\r\r\n\r\n"
+            └── unescaped: "\n\r\n\r\n"
diff --git a/test/prism/snapshots/seattlerb/heredoc_with_only_carriage_returns_windows.txt b/test/prism/snapshots/seattlerb/heredoc_with_only_carriage_returns_windows.txt
index b0f5d369..6539846f 100644
--- a/test/prism/snapshots/seattlerb/heredoc_with_only_carriage_returns_windows.txt
+++ b/test/prism/snapshots/seattlerb/heredoc_with_only_carriage_returns_windows.txt
@@ -8,4 +8,4 @@
             ├── opening_loc: (1,0)-(1,5) = "<<EOS"
             ├── content_loc: (2,0)-(5,0) = "\r\r\n\r\r\r\n\\r\r\n"
             ├── closing_loc: (5,0)-(6,0) = "EOS\r\n"
-            └── unescaped: "\r\r\n\r\r\r\n\r\r\n"
+            └── unescaped: "\r\n\r\r\n\r\n"