Batch command

client/.env.example

@@ -0,0 +1,8 @@
+# HuggingFace API Token (optional)
+# Get your token from https://huggingface.co/settings/tokens
+# Required for accessing private models or higher API rate limits
+HF_TOKEN=your_token_here
+
+# Benchmark Server URL (optional)
+# Default: http://localhost:7860
+BENCH_SERVER_URL=http://localhost:7860

client/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+.env

client/package-lock.json

@@ -8,6 +8,10 @@
       "name": "client",
       "version": "0.0.1",
       "dependencies": {
+        "@huggingface/hub": "^2.6.12",
+        "@types/prompts": "^2.4.9",
+        "dotenv": "^17.2.3",
+        "prompts": "^2.4.2",
         "table": "^6.9.0",
         "yargs": "^17.7.2"
       },
@@ -459,6 +463,49 @@
         "node": ">=18"
       }
     },
+    "node_modules/@huggingface/hub": {
+      "version": "2.6.12",
+      "resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-2.6.12.tgz",
+      "integrity": "sha512-/AZN2LAtrt4B8S83/Ru4wMorHY4NPwIYXo60SkuD6c/Mr135t1UqffD1vqcqtDYVf0hhLiyVmm1LMU1CXi8iKQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@huggingface/tasks": "^0.19.50"
+      },
+      "bin": {
+        "hfjs": "dist/cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "cli-progress": "^3.12.0"
+      }
+    },
+    "node_modules/@huggingface/tasks": {
+      "version": "0.19.50",
+      "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.19.50.tgz",
+      "integrity": "sha512-kHrfiDsJttkuwpdp7PgFiFHaK9rj+COJTIZ+221gk9vdF4B5QySW7hQT4aOGkwwZP+4qbXGhjMNGg/bxOq+LwA==",
+      "license": "MIT"
+    },
+    "node_modules/@types/node": {
+      "version": "24.7.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.7.2.tgz",
+      "integrity": "sha512-/NbVmcGTP+lj5oa4yiYxxeBjRivKQ5Ns1eSZeB99ExsEQ6rX5XYU1Zy/gGxY/ilqtD4Etx9mKyrPxZRetiahhA==",
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~7.14.0"
+      }
+    },
+    "node_modules/@types/prompts": {
+      "version": "2.4.9",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.4.9.tgz",
+      "integrity": "sha512-qTxFi6Buiu8+50/+3DGIWLHM6QuWsEKugJnnP6iv2Mc4ncxE4A/OJkjuVOA+5X0X1S/nq5VJRa8Lu+nwcvbrKA==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*",
+        "kleur": "^3.0.3"
+      }
+    },
     "node_modules/@types/yargs": {
       "version": "17.0.33",
       "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz",
@@ -525,6 +572,19 @@
         "node": ">=8"
       }
     },
+    "node_modules/cli-progress": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz",
+      "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "string-width": "^4.2.3"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/cliui": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
@@ -557,6 +617,18 @@
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "license": "MIT"
     },
+    "node_modules/dotenv": {
+      "version": "17.2.3",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
+      "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
     "node_modules/emoji-regex": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -688,12 +760,34 @@
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
       "license": "MIT"
     },
+    "node_modules/kleur": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
+      "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/lodash.truncate": {
       "version": "4.4.2",
       "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz",
       "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==",
       "license": "MIT"
     },
+    "node_modules/prompts": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz",
+      "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==",
+      "license": "MIT",
+      "dependencies": {
+        "kleur": "^3.0.3",
+        "sisteransi": "^1.0.5"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -722,6 +816,12 @@
         "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
       }
     },
+    "node_modules/sisteransi": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
+      "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==",
+      "license": "MIT"
+    },
     "node_modules/slice-ansi": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz",
@@ -815,6 +915,12 @@
         "node": ">=14.17"
       }
     },
+    "node_modules/undici-types": {
+      "version": "7.14.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.14.0.tgz",
+      "integrity": "sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==",
+      "license": "MIT"
+    },
     "node_modules/wrap-ansi": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",

client/package.json

@@ -7,6 +7,10 @@
     "cli": "tsx src/index.ts"
   },
   "dependencies": {
+    "@huggingface/hub": "^2.6.12",
+    "@types/prompts": "^2.4.9",
+    "dotenv": "^17.2.3",
+    "prompts": "^2.4.2",
     "table": "^6.9.0",
     "yargs": "^17.7.2"
   },

client/src/hf-api.ts

@@ -0,0 +1,65 @@
+/**
+ * Hugging Face API Client
+ *
+ * Uses the official @huggingface/hub library to search and retrieve models
+ */
+
+import { listModels } from "@huggingface/hub";
+import type { ModelEntry, PipelineType } from "@huggingface/hub";
+import { config as dotenvConfig } from "dotenv";
+
+// Load environment variables from .env file
+dotenvConfig();
+
+export interface SearchModelsOptions {
+  task?: PipelineType;
+  search?: string;
+  limit?: number;
+}
+
+/**
+ * Search models on Hugging Face Hub
+ */
+export async function searchModels(options: SearchModelsOptions = {}): Promise<ModelEntry[]> {
+  const { task, search, limit } = options;
+
+  const resultGenerator = listModels({
+    search: {
+      task: task,
+      query: search,
+      tags: ["transformers.js"]
+    },
+    limit: limit,
+    credentials: {
+      accessToken: process.env.HF_TOKEN,
+    },
+  });
+
+  const models: ModelEntry[] = [];
+  // Fetch models using the official HF Hub library
+  for await (const model of resultGenerator) {
+    // Filter out private and gated models
+    if (!model.private && !model.gated) {
+      models.push(model);
+    }
+
+    // Stop when we reach the limit (if specified)
+    if (limit !== undefined && models.length >= limit) {
+      break;
+    }
+  }
+
+  return models;
+}
+
+/**
+ * Format model for display
+ */
+export function formatModel(model: ModelEntry): string {
+  const downloads = model.downloads ? `${(model.downloads / 1000).toFixed(1)}k` : "N/A";
+  const likes = model.likes || 0;
+  const task = model.task || "unknown";
+  const name = model.name || model.id;
+
+  return `${name} | Task: ${task} | Downloads: ${downloads} | Likes: ${likes}`;
+}

client/src/index.ts

@@ -3,6 +3,9 @@
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import { table } from "table";
+import prompts from "prompts";
+import { searchModels, formatModel } from "./hf-api.js";
+import type { ModelEntry } from "@huggingface/hub";
 
 const SERVER_URL = process.env.BENCH_SERVER_URL || "http://localhost:7860";
 
@@ -264,6 +267,185 @@ yargs(hideBin(process.argv))
       }
     }
   )
+  .command(
+    "batch <task> [query]",
+    "Search HuggingFace models and submit benchmarks for them",
+    (yargs) => {
+      return yargs
+        .positional("task", {
+          describe: "Task type (e.g., feature-extraction, text-classification, fill-mask)",
+          type: "string",
+          demandOption: true,
+        })
+        .positional("query", {
+          describe: "Optional search query to filter model names",
+          type: "string",
+        })
+        .option("limit", {
+          describe: "Maximum number of models to benchmark",
+          type: "number",
+        })
+        .option("sort", {
+          describe: "Sort models by",
+          choices: ["downloads", "likes", "lastModified", "trending"] as const,
+          default: "downloads" as const,
+        })
+        .option("platform", {
+          describe: "Platform(s) to run on (can specify multiple)",
+          type: "array",
+          default: ["node"],
+        })
+        .option("mode", {
+          describe: "Cache mode(s) (can specify multiple)",
+          type: "array",
+          default: ["warm"],
+        })
+        .option("repeats", {
+          describe: "Number of times to repeat the benchmark",
+          type: "number",
+          default: 3,
+        })
+        .option("batch-size", {
+          describe: "Batch size(s) for inference (can specify multiple)",
+          type: "array",
+          default: [1],
+        })
+        .option("device", {
+          describe: "Device(s) for platform (can specify multiple)",
+          type: "array",
+          default: ["webgpu"],
+        })
+        .option("browser", {
+          describe: "Browser(s) for web platform (can specify multiple)",
+          type: "array",
+          default: ["chromium"],
+        })
+        .option("yes", {
+          alias: "y",
+          describe: "Skip confirmation prompt",
+          type: "boolean",
+          default: false,
+        });
+    },
+    async (argv) => {
+      console.log(`Searching for ${argv.task} models${argv.query ? ` matching "${argv.query}"` : ""}...\n`);
+
+      const models = await searchModels({
+        task: argv.task,
+        search: argv.query,
+        limit: argv.limit,
+        sort: argv.sort,
+      });
+
+      if (models.length === 0) {
+        console.log("No models found.");
+        return;
+      }
+
+      console.log(`Found ${models.length} models:\n`);
+      models.forEach((model, index) => {
+        console.log(`${index + 1}. ${formatModel(model)}`);
+      });
+
+      // Generate all combinations
+      const platforms = argv.platform as string[];
+      const modes = argv.mode as string[];
+      const batchSizes = argv.batchSize as number[];
+      const devices = argv.device as string[];
+      const browsers = argv.browser as string[];
+
+      const combinations: Array<{
+        modelId: string;
+        platform: string;
+        mode: string;
+        batchSize: number;
+        device: string;
+        browser: string;
+      }> = [];
+
+      for (const model of models) {
+        for (const platform of platforms) {
+          for (const mode of modes) {
+            for (const batchSize of batchSizes) {
+              for (const device of devices) {
+                for (const browser of browsers) {
+                  combinations.push({
+                    modelId: (model as any).name || model.id,
+                    platform,
+                    mode,
+                    batchSize,
+                    device,
+                    browser,
+                  });
+                }
+              }
+            }
+          }
+        }
+      }
+
+      console.log(`\n📊 Benchmark Plan:`);
+      console.log(`  Models: ${models.length}`);
+      console.log(`  Platforms: ${platforms.join(", ")}`);
+      console.log(`  Modes: ${modes.join(", ")}`);
+      console.log(`  Batch Sizes: ${batchSizes.join(", ")}`);
+      console.log(`  Devices: ${devices.join(", ")}`);
+      console.log(`  Browsers: ${browsers.join(", ")}`);
+      console.log(`  Total benchmarks: ${combinations.length}`);
+
+      // Ask for confirmation unless -y flag is used
+      if (!argv.yes) {
+        const response = await prompts({
+          type: "confirm",
+          name: "proceed",
+          message: `Proceed with submitting ${combinations.length} benchmark(s)?`,
+          initial: true,
+        });
+
+        if (!response.proceed) {
+          console.log("\nCancelled.");
+          return;
+        }
+      }
+
+      console.log(`\nSubmitting ${combinations.length} benchmarks...`);
+
+      const submitted: string[] = [];
+      const failed: Array<{ combo: string; error: string }> = [];
+
+      for (const combo of combinations) {
+        try {
+          const options: SubmitOptions = {
+            modelId: combo.modelId,
+            task: argv.task,
+            platform: combo.platform as "node" | "web",
+            mode: combo.mode as "warm" | "cold",
+            repeats: argv.repeats,
+            batchSize: combo.batchSize,
+            device: combo.device,
+            browser: combo.browser as "chromium" | "firefox" | "webkit",
+          };
+
+          const result = await submitBenchmark(options);
+          const desc = `${combo.modelId} [${combo.platform}/${combo.device}/${combo.mode}/b${combo.batchSize}]`;
+          submitted.push(desc);
+          console.log(`✓ Queued: ${desc} (${result.id})`);
+        } catch (error: any) {
+          const desc = `${combo.modelId} [${combo.platform}/${combo.device}]`;
+          failed.push({ combo: desc, error: error.message });
+          console.log(`✗ Failed: ${desc} - ${error.message}`);
+        }
+      }
+
+      console.log(`\n📊 Summary:`);
+      console.log(`  ✓ Submitted: ${submitted.length}`);
+      console.log(`  ✗ Failed: ${failed.length}`);
+
+      if (submitted.length > 0) {
+        console.log(`\nCheck status with: bench-client queue`);
+      }
+    }
+  )
   .demandCommand(1, "You need to specify a command")
   .help()
   .alias("h", "help")