Repository: shikijs/shiki
Pull request #916 (2025-02-03 21:46:59)

feat(engine-js): lazy compile extremely long patterns
- Bump `oniguruma-to-es` from v2.3.0 to v3.1.0, and migrate for breaking changes.
- Lazy compile patterns with a transpiled length ≥ 3000.
  - For precompiled grammars, this should improve initialization time if the grammar includes extremely long patterns, and improve overall time if the lazy-compiled regexes aren't always used.

netlify[bot]: ### <span aria-hidden="true">👷</span> Deploy Preview for *shiki-matsu* processing.


|  Name | Link |
|:-:|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | b8d0def359bf3e674523a573533e3d87fd0e05f5 |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/shiki-matsu/deploys/67a139566a6f4000084bd657 |
netlify[bot]: ### <span aria-hidden="true">👷</span> Deploy Preview for *shiki-next* processing.


|  Name | Link |
|:-:|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | b8d0def359bf3e674523a573533e3d87fd0e05f5 |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/shiki-next/deploys/67a139565750020008174eb7 |

Patch:
diff --git a/docs/references/engine-js-compat.md b/docs/references/engine-js-compat.md
index 49887c9d..600fdbee 100644
--- a/docs/references/engine-js-compat.md
+++ b/docs/references/engine-js-compat.md
@@ -2,20 +2,20 @@
 
 Compatibility reference of all built-in grammars with the [JavaScript RegExp engine](/guide/regex-engines#javascript-regexp-engine).
 
-> Generated on Tuesday, January 21, 2025
+> Generated on Monday, February 3, 2025
 >
-> Version `2.0.3`
+> Version `2.2.0`
 >
-> Runtime: Node.js v22.11.0
+> Runtime: Node.js v22.13.1
 
 ## Report Summary
 
 |                 |                       Count |
 | :-------------- | --------------------------: |
 | Total Languages |                         219 |
-| Supported       | [214](#supported-languages) |
+| Supported       | [215](#supported-languages) |
 | Mismatched      |  [0](#mismatched-languages) |
-| Unsupported     | [5](#unsupported-languages) |
+| Unsupported     | [4](#unsupported-languages) |
 
 ## Supported Languages
 
@@ -56,6 +56,7 @@ In some edge cases, it's not guaranteed that the highlighting will be 100% the s
 | cmake              | ✅ OK           |                23 |               - |      |
 | cobol              | ✅ OK           |               868 |               - |      |
 | codeowners         | ✅ OK           |                 4 |               - |      |
+| codeql             | ✅ OK           |               151 |               - |      |
 | coffee             | ✅ OK           |               471 |               - |      |
 | common-lisp        | ✅ OK           |                60 |               - |      |
 | coq                | ✅ OK           |                26 |               - |      |
@@ -259,7 +260,6 @@ Languages that throw with the JavaScript RegExp engine, either because they cont
 
 | Language   | Highlight Match | Patterns Parsable | Patterns Failed | Diff |
 | ---------- | :-------------- | ----------------: | --------------: | ---: |
-| codeql     | ✅ OK           |               150 |               1 |      |
 | csharp     | ❌ Error        |               312 |               1 |  137 |
 | purescript | ❌ Error        |                72 |               1 |      |
 | razor      | ❌ Error        |               961 |               1 |      |
diff --git a/packages/engine-javascript/src/engine-compile.ts b/packages/engine-javascript/src/engine-compile.ts
index 4346e237..42c21383 100644
--- a/packages/engine-javascript/src/engine-compile.ts
+++ b/packages/engine-javascript/src/engine-compile.ts
@@ -1,5 +1,5 @@
 import type { RegexEngine } from '@shikijs/types'
-import type { OnigurumaToEsOptions } from 'oniguruma-to-es'
+import type { ToRegExpOptions } from 'oniguruma-to-es'
 import type { JavaScriptRegexScannerOptions } from './scanner'
 import { toRegExp } from 'oniguruma-to-es'
 import { JavaScriptScanner } from './scanner'
@@ -25,12 +25,15 @@ export interface JavaScriptRegexEngineOptions extends JavaScriptRegexScannerOpti
 /**
  * The default regex constructor for the JavaScript RegExp engine.
  */
-export function defaultJavaScriptRegexConstructor(pattern: string, options?: OnigurumaToEsOptions): RegExp {
+export function defaultJavaScriptRegexConstructor(pattern: string, options?: ToRegExpOptions): RegExp {
   return toRegExp(
     pattern,
     {
       global: true,
       hasIndices: true,
+      // This has no benefit for the standard JS engine, but it avoids a perf penalty for
+      // precompiled grammars when constructing extremely long patterns that aren't always used
+      lazyCompileLength: 3000,
       rules: {
         // Needed since TextMate grammars merge backrefs across patterns
         allowOrphanBackrefs: true,
diff --git a/packages/langs-precompiled/scripts/langs.ts b/packages/langs-precompiled/scripts/langs.ts
index d21bd775..471640b1 100644
--- a/packages/langs-precompiled/scripts/langs.ts
+++ b/packages/langs-precompiled/scripts/langs.ts
@@ -126,7 +126,7 @@ export function toJsLiteral(value: any, seen = new Set()): string {
   }
 
   if (value instanceof EmulatedRegExp) {
-    return `/*@__PURE__*/ new EmulatedRegExp(${JSON.stringify(value.rawArgs.pattern)},${JSON.stringify(value.rawArgs.flags)},${JSON.stringify(value.rawArgs.options)})`
+    return `/*@__PURE__*/ new EmulatedRegExp(${JSON.stringify(value.source)},"${value.flags}",${JSON.stringify(value.rawOptions)})`
   }
 
   // RegExp
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index da7570e7..a9a177a8 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -151,8 +151,8 @@ catalogs:
       specifier: ^1.1.4
       version: 1.1.4
     oniguruma-to-es:
-      specifier: ^2.3.0
-      version: 2.3.0
+      specifier: ^3.1.0
+      version: 3.1.0
     picocolors:
       specifier: ^1.1.1
       version: 1.1.1
@@ -587,7 +587,7 @@ importers:
         version: 10.0.1
       oniguruma-to-es:
         specifier: 'catalog:'
-        version: 2.3.0
+        version: 3.1.0
 
   packages/engine-oniguruma:
     dependencies:
@@ -619,7 +619,7 @@ importers:
         version: link:../types
       oniguruma-to-es:
         specifier: 'catalog:'
-        version: 2.3.0
+        version: 3.1.0
     devDependencies:
       tm-grammars:
         specifier: 'catalog:'
@@ -4139,8 +4139,8 @@ packages:
     resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==}
     engines: {node: '>=18'}
 
-  oniguruma-to-es@2.3.0:
-    resolution: {integrity: sha512-bwALDxriqfKGfUufKGGepCzu9x7nJQuoRoAFp4AnwehhC2crqrDIAP/uN2qdlsAvSMpeRC3+Yzhqc7hLmle5+g==}
+  oniguruma-to-es@3.1.0:
+    resolution: {integrity: sha512-BJ3Jy22YlgejHSO7Fvmz1kKazlaPmRSUH+4adTDUS/dKQ4wLxI+gALZ8updbaux7/m7fIlpgOZ5fp/Inq5jUAw==}
 
   optionator@0.9.4:
     resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==}
@@ -4531,14 +4531,14 @@ packages:
     resolution: {integrity: sha512-J8rn6v4DBb2nnFqkqwy6/NnTYMcgLA+sLr0iIO41qpv0n+ngb7ksag2tMRl0inb1bbO/esUwzW1vbJi7K0sI0g==}
     engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0}
 
-  regex-recursion@5.1.1:
-    resolution: {integrity: sha512-ae7SBCbzVNrIjgSbh7wMznPcQel1DNlDtzensnFxpiNpXt1U2ju/bHugH422r+4LAVS1FpW1YCwilmnNsjum9w==}
+  regex-recursion@6.0.2:
+    resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==}
 
   regex-utilities@2.3.0:
     resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==}
 
-  regex@5.1.1:
-    resolution: {integrity: sha512-dN5I359AVGPnwzJm2jN1k0W9LPZ+ePvoOeVMMfqIMFz53sSwXkxaJoxr50ptnsC771lK95BnTrVSZxq0b9yCGw==}
+  regex@6.0.1:
+    resolution: {integrity: sha512-uorlqlzAKjKQZ5P+kTJr3eeJGSVroLKoHmquUj4zHWuR+hEyNqlXsSKlYYF5F4NI6nl7tWCs0apKJ0lmfsXAPA==}
 
   regexp-ast-analysis@0.7.1:
     resolution: {integrity: sha512-sZuz1dYW/ZsfG17WSAG7eS85r5a0dDsvg+7BiiYR5o6lKCAtUrEwdmRmaGF6rwVj3LcmAeYkOWKEPlbPzN3Y3A==}
@@ -9176,11 +9176,11 @@ snapshots:
     dependencies:
       mimic-function: 5.0.1
 
-  oniguruma-to-es@2.3.0:
+  oniguruma-to-es@3.1.0:
     dependencies:
       emoji-regex-xs: 1.0.0
-      regex: 5.1.1
-      regex-recursion: 5.1.1
+      regex: 6.0.1
+      regex-recursion: 6.0.2
 
   optionator@0.9.4:
     dependencies:
@@ -9530,14 +9530,13 @@ snapshots:
     dependencies:
       '@eslint-community/regexpp': 4.12.1
 
-  regex-recursion@5.1.1:
+  regex-recursion@6.0.2:
     dependencies:
-      regex: 5.1.1
       regex-utilities: 2.3.0
 
   regex-utilities@2.3.0: {}
 
-  regex@5.1.1:
+  regex@6.0.1:
     dependencies:
       regex-utilities: 2.3.0
 
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 3a0ecaca..945a8687 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -57,7 +57,7 @@ catalog:
   monaco-editor-core: ^0.52.2
   ofetch: ^1.4.1
   ohash: ^1.1.4
-  oniguruma-to-es: ^2.3.0
+  oniguruma-to-es: ^3.1.0
   picocolors: ^1.1.1
   pinia: ^2.3.1
   pnpm: ^9.15.4


Test patch:
diff --git a/packages/langs-precompiled/scripts/__snapshots__/precompile.test.ts.snap b/packages/langs-precompiled/scripts/__snapshots__/precompile.test.ts.snap
index aa363561..601e3f4b 100644
--- a/packages/langs-precompiled/scripts/__snapshots__/precompile.test.ts.snap
+++ b/packages/langs-precompiled/scripts/__snapshots__/precompile.test.ts.snap
@@ -92,7 +92,7 @@ exports[`precompile 1`] = `
         },
       },
       end: /*@__PURE__*/ new EmulatedRegExp("^(?=\\\\P{space})|(?!^)", "dgv", {
-        strategy: "search_start_clip",
+        strategy: "clip_search",
       }),
       patterns: [
         {
@@ -113,7 +113,7 @@ exports[`precompile 1`] = `
         "1": { name: "punctuation.whitespace.comment.leading.yaml" },
       },
       end: /*@__PURE__*/ new EmulatedRegExp("(?!^)", "dgv", {
-        strategy: "search_start_clip",
+        strategy: "clip_search",
       }),
       patterns: [
         {
@@ -378,7 +378,7 @@ exports[`precompile 1`] = `
     property: {
       begin: /(?=!|&)/dgv,
       end: /*@__PURE__*/ new EmulatedRegExp("(?!^)", "dgv", {
-        strategy: "search_start_clip",
+        strategy: "clip_search",
       }),
       name: "meta.property.yaml",
       patterns: [