Repository: ast-grep/ast-grep (language: rust)

Issue:
reduce napi binary size by removing languages



Discussion / hints:

Fix patch:
diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.toml
index c5e253c260..60f25e402c 100644
--- a/crates/language/Cargo.toml
+++ b/crates/language/Cargo.toml
@@ -64,4 +64,10 @@ builtin-parser = [
   "tree-sitter-typescript",
   "tree-sitter-yaml",
 ]
+napi-lang = [
+  "tree-sitter-css",
+  "tree-sitter-html",
+  "tree-sitter-javascript",
+  "tree-sitter-typescript",
+]
 default = ["builtin-parser"]
\ No newline at end of file
diff --git a/crates/language/src/parsers.rs b/crates/language/src/parsers.rs
index 00ba366724..f0562226d0 100644
--- a/crates/language/src/parsers.rs
+++ b/crates/language/src/parsers.rs
@@ -26,6 +26,21 @@ macro_rules! into_lang {
   };
 }
 
+#[cfg(any(feature = "builtin-parser", feature = "napi-lang"))]
+macro_rules! into_napi_lang {
+  ($lang: path) => {
+    $lang.into()
+  };
+}
+#[cfg(not(any(feature = "builtin-parser", feature = "napi-lang")))]
+macro_rules! into_napi_lang {
+  ($lang: path) => {
+    unimplemented!(
+      "tree-sitter parser is not implemented when feature flag [builtin-parser] is off."
+    )
+  };
+}
+
 use ast_grep_core::language::TSLanguage;
 
 pub fn language_bash() -> TSLanguage {
@@ -41,7 +56,7 @@ pub fn language_c_sharp() -> TSLanguage {
   into_lang!(tree_sitter_c_sharp)
 }
 pub fn language_css() -> TSLanguage {
-  into_lang!(tree_sitter_css)
+  into_napi_lang!(tree_sitter_css::LANGUAGE)
 }
 pub fn language_elixir() -> TSLanguage {
   into_lang!(tree_sitter_elixir)
@@ -53,13 +68,13 @@ pub fn language_haskell() -> TSLanguage {
   into_lang!(tree_sitter_haskell)
 }
 pub fn language_html() -> TSLanguage {
-  into_lang!(tree_sitter_html)
+  into_napi_lang!(tree_sitter_html::LANGUAGE)
 }
 pub fn language_java() -> TSLanguage {
   into_lang!(tree_sitter_java)
 }
 pub fn language_javascript() -> TSLanguage {
-  into_lang!(tree_sitter_javascript)
+  into_napi_lang!(tree_sitter_javascript::LANGUAGE)
 }
 pub fn language_json() -> TSLanguage {
   into_lang!(tree_sitter_json)
@@ -89,10 +104,10 @@ pub fn language_swift() -> TSLanguage {
   into_lang!(tree_sitter_swift)
 }
 pub fn language_tsx() -> TSLanguage {
-  into_lang!(tree_sitter_typescript, LANGUAGE_TSX)
+  into_napi_lang!(tree_sitter_typescript::LANGUAGE_TSX)
 }
 pub fn language_typescript() -> TSLanguage {
-  into_lang!(tree_sitter_typescript, LANGUAGE_TYPESCRIPT)
+  into_napi_lang!(tree_sitter_typescript::LANGUAGE_TYPESCRIPT)
 }
 pub fn language_yaml() -> TSLanguage {
   into_lang!(tree_sitter_yaml)
diff --git a/crates/napi/Cargo.toml b/crates/napi/Cargo.toml
index c2174a3cf3..2edb6a0eba 100644
--- a/crates/napi/Cargo.toml
+++ b/crates/napi/Cargo.toml
@@ -13,7 +13,7 @@ edition.workspace = true
 [dependencies]
 ast-grep-core.workspace = true
 ast-grep-config.workspace = true
-ast-grep-language.workspace = true
+ast-grep-language = { path = "../language", features = ["napi-lang"], default-features = false }
 ast-grep-dynamic.workspace = true
 
 napi = { version = "2.16.4", features = ["serde-json", "napi4", "error_anyhow"] }
diff --git a/crates/napi/scripts/constants.ts b/crates/napi/scripts/constants.ts
index ad8c5fa707..4eca529e7b 100644
--- a/crates/napi/scripts/constants.ts
+++ b/crates/napi/scripts/constants.ts
@@ -1,36 +1,11 @@
 import { Lang } from '..'
 
-export const languageNodeTypesTagVersionOverrides: Partial<
-  Record<Lang, string>
-> = {
-  // The latest version is not tagged yet, so we have to use the latest available tag
-  [Lang.Kotlin]: '0.3.8',
-}
-
 export const languagesCrateNames: Record<Lang, string> = {
   [Lang.JavaScript]: 'tree-sitter-javascript',
   [Lang.TypeScript]: 'tree-sitter-typescript',
   [Lang.Tsx]: 'tree-sitter-typescript',
-  [Lang.Java]: 'tree-sitter-java',
-  [Lang.Python]: 'tree-sitter-python',
-  [Lang.Rust]: 'tree-sitter-rust',
-  [Lang.C]: 'tree-sitter-c',
-  [Lang.Cpp]: 'tree-sitter-cpp',
-  [Lang.Go]: 'tree-sitter-go',
   [Lang.Html]: 'tree-sitter-html',
   [Lang.Css]: 'tree-sitter-css',
-  [Lang.Json]: 'tree-sitter-json',
-  [Lang.CSharp]: 'tree-sitter-c-sharp',
-  [Lang.Ruby]: 'tree-sitter-ruby',
-  [Lang.Php]: 'tree-sitter-php',
-  [Lang.Elixir]: 'tree-sitter-elixir',
-  [Lang.Kotlin]: 'tree-sitter-kotlin',
-  [Lang.Swift]: 'tree-sitter-swift',
-  [Lang.Haskell]: 'tree-sitter-haskell',
-  [Lang.Scala]: 'tree-sitter-scala',
-  [Lang.Lua]: 'tree-sitter-lua',
-  [Lang.Bash]: 'tree-sitter-bash',
-  [Lang.Yaml]: 'tree-sitter-yaml',
 }
 
 export const languagesNodeTypesUrls = {
@@ -40,43 +15,8 @@ export const languagesNodeTypesUrls = {
     'https://raw.githubusercontent.com/tree-sitter/tree-sitter-typescript/refs/tags/{{TAG}}/typescript/src/node-types.json',
   [Lang.Tsx]:
     'https://raw.githubusercontent.com/tree-sitter/tree-sitter-typescript/refs/tags/{{TAG}}/tsx/src/node-types.json',
-  [Lang.Java]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-java/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Python]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-python/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Rust]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-rust/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.C]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-c/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Cpp]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-cpp/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Go]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-go/refs/tags/{{TAG}}/src/node-types.json',
   [Lang.Html]:
     'https://raw.githubusercontent.com/tree-sitter/tree-sitter-html/refs/tags/{{TAG}}/src/node-types.json',
   [Lang.Css]:
     'https://raw.githubusercontent.com/tree-sitter/tree-sitter-css/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Json]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-json/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.CSharp]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-c-sharp/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Ruby]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-ruby/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Php]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-php/refs/tags/{{TAG}}/php/src/node-types.json',
-  [Lang.Elixir]:
-    'https://raw.githubusercontent.com/elixir-lang/tree-sitter-elixir/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Kotlin]:
-    'https://raw.githubusercontent.com/fwcd/tree-sitter-kotlin/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Haskell]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-haskell/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Scala]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-scala/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Bash]:
-    'https://raw.githubusercontent.com/tree-sitter/tree-sitter-bash/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Yaml]:
-    'https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-yaml/refs/tags/{{TAG}}/src/node-types.json',
-  [Lang.Lua]:
-    'https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-lua/refs/tags/{{TAG}}/src/node-types.json',
-  // Not available for Swift - They don't have node-types.json in their repo contents
 }
\ No newline at end of file
diff --git a/crates/napi/scripts/generateTypes.ts b/crates/napi/scripts/generateTypes.ts
index 96b5b669ea..de847a9e5f 100644
--- a/crates/napi/scripts/generateTypes.ts
+++ b/crates/napi/scripts/generateTypes.ts
@@ -4,7 +4,6 @@ import path from 'node:path'
 import type { Lang } from '../index'
 import type { NodeType } from '../types/staticTypes'
 import {
-  languageNodeTypesTagVersionOverrides,
   languagesCrateNames,
   languagesNodeTypesUrls,
 } from './constants'
@@ -77,8 +76,7 @@ async function generateLangNodeTypes() {
       const treeSitterCrateName = languagesCrateNames[lang]
       const cargoVersion =
         parsedCargoToml.dependencies[treeSitterCrateName].version
-      const tag =
-        languageNodeTypesTagVersionOverrides[lang] ?? `v${cargoVersion}`
+      const tag = `v${cargoVersion}`
       const url = urlTemplate.replace('{{TAG}}', tag)
       const nodeTypesResponse = await fetch(url)
       const nodeTypes = (await nodeTypesResponse.json()) as NodeType[]
@@ -101,4 +99,4 @@ async function generateLangNodeTypes() {
 generateLangNodeTypes().catch(error => {
   console.error('Error:', error)
   process.exit(1)
-})
+})
\ No newline at end of file
diff --git a/crates/napi/src/napi_lang.rs b/crates/napi/src/napi_lang.rs
index e9b8b6a647..c534b58785 100644
--- a/crates/napi/src/napi_lang.rs
+++ b/crates/napi/src/napi_lang.rs
@@ -23,24 +23,6 @@ pub enum Lang {
   Tsx,
   Css,
   TypeScript,
-  Bash,
-  C,
-  Cpp,
-  CSharp,
-  Go,
-  Elixir,
-  Haskell,
-  Java,
-  Json,
-  Kotlin,
-  Lua,
-  Php,
-  Python,
-  Ruby,
-  Rust,
-  Scala,
-  Swift,
-  Yaml,
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
@@ -52,7 +34,11 @@ pub enum NapiLang {
 
 impl NapiLang {
   fn all_langs() -> Vec<Self> {
-    let builtin = SupportLang::all_langs().iter().copied().map(Self::Builtin);
+    use SupportLang as S;
+    let builtin = [S::Html, S::JavaScript, S::Tsx, S::Css, S::TypeScript]
+      .iter()
+      .copied()
+      .map(Self::Builtin);
     let customs = DynamicLang::all_langs().into_iter().map(Self::Custom);
     builtin.chain(customs).collect()
   }
@@ -103,9 +89,15 @@ impl Debug for NapiLang {
 impl FromStr for NapiLang {
   type Err = Error;
   fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
-    if let Ok(b) = SupportLang::from_str(s) {
-      Ok(NapiLang::Builtin(b))
-    } else if let Ok(c) = DynamicLang::from_str(s) {
+    use SupportLang as S;
+    // only support frontend languages to reduce binary size
+    if let Ok(b) = S::from_str(s) {
+      if matches!(b, S::Css | S::Html | S::JavaScript | S::Tsx | S::TypeScript) {
+        return Ok(NapiLang::Builtin(b));
+      }
+    }
+
+    if let Ok(c) = DynamicLang::from_str(s) {
       Ok(NapiLang::Custom(c))
     } else {
       Err(anyhow!(format!("{s} is not supported in napi")))
diff --git a/crates/napi/types/lang.d.ts b/crates/napi/types/lang.d.ts
index 19a26e5f21..188b5e248e 100644
--- a/crates/napi/types/lang.d.ts
+++ b/crates/napi/types/lang.d.ts
@@ -4,24 +4,6 @@ export enum Lang {
   Tsx = 'Tsx',
   Css = 'Css',
   TypeScript = 'TypeScript',
-  Bash = 'Bash',
-  C = 'C',
-  Cpp = 'Cpp',
-  CSharp = 'CSharp',
-  Go = 'Go',
-  Elixir = 'Elixir',
-  Haskell = 'Haskell',
-  Java = 'Java',
-  Json = 'Json',
-  Kotlin = 'Kotlin',
-  Lua = 'Lua',
-  Php = 'Php',
-  Python = 'Python',
-  Ruby = 'Ruby',
-  Rust = 'Rust',
-  Scala = 'Scala',
-  Swift = 'Swift',
-  Yaml = 'Yaml',
 }
 
 type CustomLang = string & {}


Test patch:
diff --git a/crates/napi/__test__/custom.spec.ts b/crates/napi/__test__/custom.spec.ts
index 5999f409fd..918fc0fd1b 100644
--- a/crates/napi/__test__/custom.spec.ts
+++ b/crates/napi/__test__/custom.spec.ts
@@ -3,6 +3,7 @@ import test from 'ava'
 import {
   registerDynamicLanguage,
   parse,
+  findInFiles,
 } from '../index'
 
 const { platform, arch } = process
@@ -13,18 +14,18 @@ const canTestDynamicLang = isAppleSilicon || isX64Linux
 
 if (isAppleSilicon) {
   registerDynamicLanguage({
-    myjson: {
+    json: {
       libraryPath: "../../benches/fixtures/json-mac.so",
       languageSymbol: "tree_sitter_json",
-      extensions: ["myjson"],
+      extensions: ["json"],
     }
   })
 } else if (isX64Linux) {
   registerDynamicLanguage({
-    myjson: {
+    json: {
       libraryPath: "../../benches/fixtures/json-linux.so",
       languageSymbol: "tree_sitter_json",
-      extensions: ["myjson"],
+      extensions: ["json"],
     }
   })
 }
@@ -34,11 +35,28 @@ test('test load custom lang', t => {
     t.pass('This test is not available on this platform')
     return
   }
-  const sg = parse('myjson', '{"test": 123}')
+  const sg = parse('json', '{"test": 123}')
   const root = sg.root()
   const node = root.find("123")!
   t.truthy(node)
   t.is(node.kind(), 'number')
   const no = root.find("456")
   t.falsy(no)
+})
+
+test('discover file', async t => {
+  await findInFiles('json', {
+    paths: ['../'],
+    matcher: {
+      rule: {
+        kind: 'string'
+      }
+    }
+  }, (error, nodes) => {
+    t.falsy(error)
+    t.truthy(nodes)
+    t.is(nodes[0].kind(), 'string')
+    const file = nodes[0].getRoot().filename()
+    t.assert(file.endsWith('.json'))
+  })
 })
\ No newline at end of file
diff --git a/crates/napi/__test__/index.spec.ts b/crates/napi/__test__/index.spec.ts
index f31550f863..6b422ecefa 100644
--- a/crates/napi/__test__/index.spec.ts
+++ b/crates/napi/__test__/index.spec.ts
@@ -5,11 +5,7 @@ import {
   ts,
   tsx,
   html,
-  Lang,
   parseFiles,
-  parseAsync,
-  findInFiles,
-  parse as parseWithLang,
 } from '../index'
 const { parse, kind } = js
 const parseMulti = countedPromise(parseFiles)
@@ -384,42 +380,6 @@ function countedPromise<F extends (t: any, cb: any) => Promise<number>>(
   }
 }
 
-test('parse python', t => {
-  const sg = parseWithLang(Lang.Python, 'print("hello world")')
-  const node = sg.root().find('print')
-  t.deepEqual(node!.range(), {
-    start: { line: 0, column: 0, index: 0 },
-    end: { line: 0, column: 5, index: 5 },
-  })
-})
-
-test('parse python async', async t => {
-  const sg = await parseAsync(Lang.Python, 'print("hello world")')
-  const node = sg.root().find('print')
-  t.deepEqual(node!.range(), {
-    start: { line: 0, column: 0, index: 0 },
-    end: { line: 0, column: 5, index: 5 },
-  })
-})
-
-test('find rust', async t => {
-  let changed = false
-  const counted = countedPromise((t, cb) => findInFiles(Lang.Rust, t, cb))
-  const num = await counted(
-    {
-      paths: ['./'],
-      matcher: {
-        rule: { pattern: 'ast_grep_core' },
-      },
-    },
-    () => {
-      changed = true
-    },
-  )
-  t.assert(changed)
-  t.assert(num > 0)
-})
-
 test('find node by range', t => {
   const sg = parse(
     'const message = `This is a multiline message with unicode characters\nÜbergang\nدرود\n🙌\n`;',
@@ -460,7 +420,7 @@ test('check if a node follows another using pattern', t => {
 test('check if a node follows another using config', t => {
   const sg = parse('const a = 1; const b = 2;')
   const match = sg.root().find('const a = 1')
-  t.assert(match!.follows({ rule: { pattern: 'const b = 2' }}) === false) 
+  t.assert(match!.follows({ rule: { pattern: 'const b = 2' }}) === false)
   t.assert(sg.root().find('const b = 2')!.follows({ rule: { pattern: 'const a = 1' }}))
 })
 
@@ -504,4 +464,4 @@ test('check if a node has another using config', t => {
   const match = sg.root().find('if (true) { $$$ }')
   t.assert(match!.has({ rule: { pattern: 'const x = 1' }}))
   t.assert(match!.has({ rule: { pattern: 'const y = 2' }}) === false)
-})
+})
\ No newline at end of file