whitphx HF Staff commited on
Commit
0310320
·
1 Parent(s): 317d5b5

Add server and its client utility

Browse files
.gitignore CHANGED
@@ -133,3 +133,6 @@ dist
133
  bench/.bench-cache
134
  bench-node/.bench-cache
135
  bench-web/.transformers-cache
 
 
 
 
133
  bench/.bench-cache
134
  bench-node/.bench-cache
135
  bench-web/.transformers-cache
136
+
137
+ # Benchmark result files
138
+ bench/benchmark-results.jsonl
bench/package-lock.json CHANGED
@@ -8,7 +8,9 @@
8
  "name": "bench",
9
  "version": "0.0.2",
10
  "dependencies": {
11
- "@huggingface/transformers": "^3.7.4"
 
 
12
  },
13
  "devDependencies": {
14
  "@playwright/test": "^1.55.1",
@@ -470,6 +472,18 @@
470
  "node": ">=18"
471
  }
472
  },
 
 
 
 
 
 
 
 
 
 
 
 
473
  "node_modules/@huggingface/jinja": {
474
  "version": "0.5.1",
475
  "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.1.tgz",
@@ -1592,6 +1606,15 @@
1592
  "url": "https://github.com/sponsors/ljharb"
1593
  }
1594
  },
 
 
 
 
 
 
 
 
 
1595
  "node_modules/json-stringify-safe": {
1596
  "version": "5.0.1",
1597
  "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
 
8
  "name": "bench",
9
  "version": "0.0.2",
10
  "dependencies": {
11
+ "@hono/node-server": "^1.19.5",
12
+ "@huggingface/transformers": "^3.7.4",
13
+ "hono": "^4.9.10"
14
  },
15
  "devDependencies": {
16
  "@playwright/test": "^1.55.1",
 
472
  "node": ">=18"
473
  }
474
  },
475
+ "node_modules/@hono/node-server": {
476
+ "version": "1.19.5",
477
+ "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.5.tgz",
478
+ "integrity": "sha512-iBuhh+uaaggeAuf+TftcjZyWh2GEgZcVGXkNtskLVoWaXhnJtC5HLHrU8W1KHDoucqO1MswwglmkWLFyiDn4WQ==",
479
+ "license": "MIT",
480
+ "engines": {
481
+ "node": ">=18.14.1"
482
+ },
483
+ "peerDependencies": {
484
+ "hono": "^4"
485
+ }
486
+ },
487
  "node_modules/@huggingface/jinja": {
488
  "version": "0.5.1",
489
  "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.1.tgz",
 
1606
  "url": "https://github.com/sponsors/ljharb"
1607
  }
1608
  },
1609
+ "node_modules/hono": {
1610
+ "version": "4.9.10",
1611
+ "resolved": "https://registry.npmjs.org/hono/-/hono-4.9.10.tgz",
1612
+ "integrity": "sha512-AlI15ijFyKTXR7eHo7QK7OR4RoKIedZvBuRjO8iy4zrxvlY5oFCdiRG/V/lFJHCNXJ0k72ATgnyzx8Yqa5arug==",
1613
+ "license": "MIT",
1614
+ "engines": {
1615
+ "node": ">=16.9.0"
1616
+ }
1617
+ },
1618
  "node_modules/json-stringify-safe": {
1619
  "version": "5.0.1",
1620
  "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
bench/package.json CHANGED
@@ -7,13 +7,17 @@
7
  "bench": "tsx src/index.ts",
8
  "bench:node": "tsx src/node/index.ts",
9
  "bench:web": "tsx src/web/cli.ts",
 
 
 
10
  "dev": "vite",
11
  "build": "tsc -p tsconfig.json",
12
- "preview": "vite preview",
13
- "bench:install": "playwright install"
14
  },
15
  "dependencies": {
16
- "@huggingface/transformers": "^3.7.4"
 
 
17
  },
18
  "devDependencies": {
19
  "@playwright/test": "^1.55.1",
 
7
  "bench": "tsx src/index.ts",
8
  "bench:node": "tsx src/node/index.ts",
9
  "bench:web": "tsx src/web/cli.ts",
10
+ "bench:install": "playwright install",
11
+ "server": "tsx src/server/index.ts",
12
+ "server:dev": "tsx watch src/server/index.ts",
13
  "dev": "vite",
14
  "build": "tsc -p tsconfig.json",
15
+ "preview": "vite preview"
 
16
  },
17
  "dependencies": {
18
+ "@hono/node-server": "^1.19.5",
19
+ "@huggingface/transformers": "^3.7.4",
20
+ "hono": "^4.9.10"
21
  },
22
  "devDependencies": {
23
  "@playwright/test": "^1.55.1",
bench/src/server/index.ts ADDED
@@ -0,0 +1,241 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { Hono } from "hono";
2
+ import { cors } from "hono/cors";
3
+ import { serve } from "@hono/node-server";
4
+ import { BenchmarkQueue, BenchmarkRequest } from "./queue.js";
5
+ import { BenchmarkStorage } from "./storage.js";
6
+ import { randomUUID } from "crypto";
7
+
8
+ const app = new Hono();
9
+ const queue = new BenchmarkQueue();
10
+ const storage = new BenchmarkStorage();
11
+
12
+ // Enable CORS for development
13
+ app.use("/*", cors());
14
+
15
+ // Store completed benchmarks to file
16
+ queue.on("completed", async (benchmark) => {
17
+ try {
18
+ await storage.appendResult(benchmark);
19
+ console.log(`✓ Benchmark ${benchmark.id} saved to file`);
20
+ } catch (error) {
21
+ console.error(`✗ Failed to save benchmark ${benchmark.id}:`, error);
22
+ }
23
+ });
24
+
25
+ queue.on("failed", async (benchmark) => {
26
+ try {
27
+ await storage.appendResult(benchmark);
28
+ console.log(`✗ Failed benchmark ${benchmark.id} saved to file`);
29
+ } catch (error) {
30
+ console.error(`✗ Failed to save failed benchmark ${benchmark.id}:`, error);
31
+ }
32
+ });
33
+
34
+ // Log queue events
35
+ queue.on("added", (benchmark) => {
36
+ console.log(`📥 Added to queue: ${benchmark.id} (${benchmark.platform}/${benchmark.modelId})`);
37
+ });
38
+
39
+ queue.on("started", (benchmark) => {
40
+ console.log(`🚀 Started: ${benchmark.id}`);
41
+ });
42
+
43
+ queue.on("completed", (benchmark) => {
44
+ console.log(`✅ Completed: ${benchmark.id} in ${(benchmark.completedAt! - benchmark.startedAt!) / 1000}s`);
45
+ });
46
+
47
+ queue.on("failed", (benchmark) => {
48
+ console.log(`❌ Failed: ${benchmark.id} - ${benchmark.error}`);
49
+ });
50
+
51
+ // API Endpoints
52
+
53
+ /**
54
+ * POST /api/benchmark
55
+ * Submit a new benchmark request
56
+ */
57
+ app.post("/api/benchmark", async (c) => {
58
+ const body = await c.req.json();
59
+ const {
60
+ platform = "node",
61
+ modelId,
62
+ task,
63
+ mode = "warm",
64
+ repeats = 3,
65
+ dtype,
66
+ batchSize = 1,
67
+ device = "webgpu",
68
+ browser = "chromium",
69
+ headed = false,
70
+ } = body;
71
+
72
+ if (!modelId || !task) {
73
+ return c.json({ error: "modelId and task are required" }, 400);
74
+ }
75
+
76
+ const request: BenchmarkRequest = {
77
+ id: randomUUID(),
78
+ platform,
79
+ modelId,
80
+ task,
81
+ mode,
82
+ repeats,
83
+ dtype,
84
+ batchSize,
85
+ device,
86
+ browser,
87
+ headed,
88
+ timestamp: Date.now(),
89
+ };
90
+
91
+ queue.addBenchmark(request);
92
+
93
+ return c.json({
94
+ id: request.id,
95
+ message: "Benchmark queued",
96
+ position: queue.getQueueStatus().pending,
97
+ });
98
+ });
99
+
100
+ /**
101
+ * GET /api/benchmark/:id
102
+ * Get benchmark status/result by ID
103
+ */
104
+ app.get("/api/benchmark/:id", async (c) => {
105
+ const id = c.req.param("id");
106
+
107
+ // Check queue first (for pending/running benchmarks)
108
+ const queued = queue.getBenchmark(id);
109
+ if (queued) {
110
+ return c.json(queued);
111
+ }
112
+
113
+ // Check storage (for completed benchmarks)
114
+ const stored = await storage.getResultById(id);
115
+ if (stored) {
116
+ return c.json(stored);
117
+ }
118
+
119
+ return c.json({ error: "Benchmark not found" }, 404);
120
+ });
121
+
122
+ /**
123
+ * GET /api/benchmarks
124
+ * Get all benchmark results from storage
125
+ */
126
+ app.get("/api/benchmarks", async (c) => {
127
+ const results = await storage.getAllResults();
128
+ return c.json({
129
+ total: results.length,
130
+ results,
131
+ });
132
+ });
133
+
134
+ /**
135
+ * GET /api/queue
136
+ * Get current queue status
137
+ */
138
+ app.get("/api/queue", (c) => {
139
+ const status = queue.getQueueStatus();
140
+ const allBenchmarks = queue.getAllBenchmarks();
141
+
142
+ return c.json({
143
+ status,
144
+ queue: allBenchmarks,
145
+ });
146
+ });
147
+
148
+ /**
149
+ * DELETE /api/benchmarks
150
+ * Clear all stored results
151
+ */
152
+ app.delete("/api/benchmarks", async (c) => {
153
+ await storage.clearResults();
154
+ return c.json({ message: "All results cleared" });
155
+ });
156
+
157
+ /**
158
+ * GET /
159
+ * Simple status page
160
+ */
161
+ app.get("/", (c) => {
162
+ const status = queue.getQueueStatus();
163
+ return c.html(`
164
+ <!DOCTYPE html>
165
+ <html>
166
+ <head>
167
+ <title>Benchmark Server</title>
168
+ <style>
169
+ body { font-family: system-ui; max-width: 800px; margin: 2rem auto; padding: 0 1rem; }
170
+ h1 { color: #333; }
171
+ .status { background: #f5f5f5; padding: 1rem; border-radius: 4px; margin: 1rem 0; }
172
+ .endpoint { background: #fff; border: 1px solid #ddd; padding: 0.5rem; margin: 0.5rem 0; border-radius: 4px; }
173
+ .method { display: inline-block; width: 60px; font-weight: bold; color: #0066cc; }
174
+ code { background: #f0f0f0; padding: 2px 6px; border-radius: 3px; }
175
+ </style>
176
+ </head>
177
+ <body>
178
+ <h1>🚀 Transformers.js Benchmark Server</h1>
179
+
180
+ <div class="status">
181
+ <h2>Queue Status</h2>
182
+ <p>Pending: ${status.pending} | Running: ${status.running} | Completed: ${status.completed} | Failed: ${status.failed}</p>
183
+ </div>
184
+
185
+ <h2>API Endpoints</h2>
186
+
187
+ <div class="endpoint">
188
+ <span class="method">POST</span> <code>/api/benchmark</code> - Submit benchmark request
189
+ </div>
190
+
191
+ <div class="endpoint">
192
+ <span class="method">GET</span> <code>/api/benchmark/:id</code> - Get benchmark status/result
193
+ </div>
194
+
195
+ <div class="endpoint">
196
+ <span class="method">GET</span> <code>/api/benchmarks</code> - Get all stored results
197
+ </div>
198
+
199
+ <div class="endpoint">
200
+ <span class="method">GET</span> <code>/api/queue</code> - Get queue status
201
+ </div>
202
+
203
+ <div class="endpoint">
204
+ <span class="method">DELETE</span> <code>/api/benchmarks</code> - Clear all results
205
+ </div>
206
+
207
+ <h2>Example Request</h2>
208
+ <pre style="background: #f5f5f5; padding: 1rem; border-radius: 4px; overflow-x: auto;">
209
+ curl -X POST http://localhost:3000/api/benchmark \\
210
+ -H "Content-Type: application/json" \\
211
+ -d '{
212
+ "platform": "node",
213
+ "modelId": "Xenova/all-MiniLM-L6-v2",
214
+ "task": "feature-extraction",
215
+ "mode": "warm",
216
+ "repeats": 3,
217
+ "batchSize": 1
218
+ }'
219
+ </pre>
220
+ </body>
221
+ </html>
222
+ `);
223
+ });
224
+
225
+ const PORT = Number(process.env.PORT) || 3000;
226
+
227
+ serve({
228
+ fetch: app.fetch,
229
+ port: PORT,
230
+ }, (info) => {
231
+ console.log(`
232
+ 🚀 Benchmark Server running on http://localhost:${info.port}
233
+
234
+ API Endpoints:
235
+ POST /api/benchmark - Submit benchmark
236
+ GET /api/benchmark/:id - Get result
237
+ GET /api/benchmarks - List all results
238
+ GET /api/queue - Queue status
239
+ DELETE /api/benchmarks - Clear results
240
+ `);
241
+ });
bench/src/server/queue.ts ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { EventEmitter } from "events";
2
+ import { BenchmarkResult } from "../core/types.js";
3
+
4
+ export interface BenchmarkRequest {
5
+ id: string;
6
+ platform: "node" | "web";
7
+ modelId: string;
8
+ task: string;
9
+ mode: "warm" | "cold";
10
+ repeats: number;
11
+ dtype?: string;
12
+ batchSize: number;
13
+ device?: string;
14
+ browser?: string;
15
+ headed?: boolean;
16
+ timestamp: number;
17
+ }
18
+
19
+ export interface QueuedBenchmark extends BenchmarkRequest {
20
+ status: "pending" | "running" | "completed" | "failed";
21
+ result?: BenchmarkResult;
22
+ error?: string;
23
+ startedAt?: number;
24
+ completedAt?: number;
25
+ }
26
+
27
+ export class BenchmarkQueue extends EventEmitter {
28
+ private queue: QueuedBenchmark[] = [];
29
+ private isProcessing = false;
30
+
31
+ addBenchmark(request: BenchmarkRequest): void {
32
+ const queued: QueuedBenchmark = {
33
+ ...request,
34
+ status: "pending",
35
+ };
36
+ this.queue.push(queued);
37
+ this.emit("added", queued);
38
+ this.processQueue();
39
+ }
40
+
41
+ getBenchmark(id: string): QueuedBenchmark | undefined {
42
+ return this.queue.find((b) => b.id === id);
43
+ }
44
+
45
+ getAllBenchmarks(): QueuedBenchmark[] {
46
+ return [...this.queue];
47
+ }
48
+
49
+ getQueueStatus() {
50
+ return {
51
+ total: this.queue.length,
52
+ pending: this.queue.filter((b) => b.status === "pending").length,
53
+ running: this.queue.filter((b) => b.status === "running").length,
54
+ completed: this.queue.filter((b) => b.status === "completed").length,
55
+ failed: this.queue.filter((b) => b.status === "failed").length,
56
+ };
57
+ }
58
+
59
+ private async processQueue() {
60
+ if (this.isProcessing) return;
61
+
62
+ const pending = this.queue.find((b) => b.status === "pending");
63
+ if (!pending) return;
64
+
65
+ this.isProcessing = true;
66
+ pending.status = "running";
67
+ pending.startedAt = Date.now();
68
+
69
+ this.emit("started", pending);
70
+
71
+ try {
72
+ const result = await this.runBenchmark(pending);
73
+ pending.status = "completed";
74
+ pending.result = result;
75
+ pending.completedAt = Date.now();
76
+ this.emit("completed", pending);
77
+ } catch (error) {
78
+ pending.status = "failed";
79
+ // Capture detailed error information
80
+ if (error instanceof Error) {
81
+ pending.error = error.message;
82
+ // Log full error details to console
83
+ console.error(`\n❌ Benchmark ${pending.id} failed:`);
84
+ console.error(` Message: ${error.message}`);
85
+ if (error.stack) {
86
+ console.error(` Stack trace:\n${error.stack}`);
87
+ }
88
+ } else {
89
+ pending.error = String(error);
90
+ console.error(`\n❌ Benchmark ${pending.id} failed: ${pending.error}`);
91
+ }
92
+ pending.completedAt = Date.now();
93
+ this.emit("failed", pending);
94
+ }
95
+
96
+ this.isProcessing = false;
97
+ // Process next item
98
+ setImmediate(() => this.processQueue());
99
+ }
100
+
101
+ private async runBenchmark(request: BenchmarkRequest): Promise<BenchmarkResult> {
102
+ if (request.platform === "node") {
103
+ // Use spawn instead of dynamic import to avoid import.meta.url issues
104
+ const { spawn } = await import("child_process");
105
+
106
+ // Build command args
107
+ const args = [
108
+ "src/node/index.ts",
109
+ request.modelId,
110
+ request.task,
111
+ `--mode=${request.mode}`,
112
+ `--repeats=${request.repeats}`,
113
+ `--batch-size=${request.batchSize}`,
114
+ ];
115
+ if (request.dtype) args.push(`--dtype=${request.dtype}`);
116
+
117
+ console.log(`\n[Queue] Dispatching node benchmark with command: tsx ${args.join(' ')}`);
118
+
119
+ return new Promise((resolve, reject) => {
120
+ const proc = spawn("tsx", args, { cwd: process.cwd() });
121
+ let stdout = "";
122
+ let stderr = "";
123
+
124
+ proc.stdout.on("data", (data) => {
125
+ stdout += data.toString();
126
+ });
127
+
128
+ proc.stderr.on("data", (data) => {
129
+ stderr += data.toString();
130
+ });
131
+
132
+ proc.on("close", (code) => {
133
+ if (code !== 0) {
134
+ reject(new Error(`Benchmark failed with code ${code}: ${stderr}`));
135
+ return;
136
+ }
137
+
138
+ // Extract JSON from stdout (last JSON object)
139
+ const jsonMatch = stdout.match(/\{[\s\S]*"platform"[\s\S]*\}/);
140
+ if (jsonMatch) {
141
+ try {
142
+ const result = JSON.parse(jsonMatch[0]);
143
+ resolve(result);
144
+ } catch (e) {
145
+ reject(new Error(`Failed to parse benchmark result: ${e}`));
146
+ }
147
+ } else {
148
+ reject(new Error("No benchmark result found in output"));
149
+ }
150
+ });
151
+ });
152
+ } else {
153
+ // For web benchmarks, we'll use the CLI approach with Playwright
154
+ const { spawn } = await import("child_process");
155
+
156
+ // Build command args
157
+ const args = [
158
+ "src/web/cli.ts",
159
+ request.modelId,
160
+ request.task,
161
+ `--mode=${request.mode}`,
162
+ `--repeats=${request.repeats}`,
163
+ `--device=${request.device || "webgpu"}`,
164
+ `--batch-size=${request.batchSize}`,
165
+ ];
166
+ if (request.dtype) args.push(`--dtype=${request.dtype}`);
167
+ if (request.browser) args.push(`--browser=${request.browser}`);
168
+ if (request.headed) args.push(`--headed=true`);
169
+
170
+ console.log(`\n[Queue] Dispatching web benchmark with command: tsx ${args.join(' ')}`);
171
+
172
+ return new Promise((resolve, reject) => {
173
+ const proc = spawn("tsx", args, { cwd: process.cwd() });
174
+ let stdout = "";
175
+ let stderr = "";
176
+
177
+ proc.stdout.on("data", (data) => {
178
+ stdout += data.toString();
179
+ });
180
+
181
+ proc.stderr.on("data", (data) => {
182
+ stderr += data.toString();
183
+ });
184
+
185
+ proc.on("close", (code) => {
186
+ if (code !== 0) {
187
+ reject(new Error(`Benchmark failed with code ${code}: ${stderr}`));
188
+ return;
189
+ }
190
+
191
+ // Extract JSON from stdout (last JSON object)
192
+ const jsonMatch = stdout.match(/\{[\s\S]*"platform"[\s\S]*\}/);
193
+ if (jsonMatch) {
194
+ try {
195
+ const result = JSON.parse(jsonMatch[0]);
196
+ resolve(result);
197
+ } catch (e) {
198
+ reject(new Error(`Failed to parse benchmark result: ${e}`));
199
+ }
200
+ } else {
201
+ reject(new Error("No benchmark result found in output"));
202
+ }
203
+ });
204
+ });
205
+ }
206
+ }
207
+ }
bench/src/server/storage.ts ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import fs from "fs/promises";
2
+ import path from "path";
3
+ import { QueuedBenchmark } from "./queue.js";
4
+
5
+ export class BenchmarkStorage {
6
+ private filePath: string;
7
+
8
+ constructor(filePath: string = "./benchmark-results.jsonl") {
9
+ this.filePath = path.resolve(filePath);
10
+ }
11
+
12
+ async appendResult(benchmark: QueuedBenchmark): Promise<void> {
13
+ const line = JSON.stringify(benchmark) + "\n";
14
+ await fs.appendFile(this.filePath, line, "utf-8");
15
+ }
16
+
17
+ async getAllResults(): Promise<QueuedBenchmark[]> {
18
+ try {
19
+ const content = await fs.readFile(this.filePath, "utf-8");
20
+ const lines = content.trim().split("\n").filter(line => line.length > 0);
21
+ return lines.map(line => JSON.parse(line));
22
+ } catch (error: any) {
23
+ if (error.code === "ENOENT") {
24
+ return []; // File doesn't exist yet
25
+ }
26
+ throw error;
27
+ }
28
+ }
29
+
30
+ async getResultById(id: string): Promise<QueuedBenchmark | undefined> {
31
+ const results = await this.getAllResults();
32
+ return results.find(r => r.id === id);
33
+ }
34
+
35
+ async clearResults(): Promise<void> {
36
+ try {
37
+ await fs.unlink(this.filePath);
38
+ } catch (error: any) {
39
+ if (error.code !== "ENOENT") {
40
+ throw error;
41
+ }
42
+ }
43
+ }
44
+ }
client/README.md ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Benchmark Client
2
+
3
+ CLI client for interacting with the Transformers.js benchmark server.
4
+
5
+ ## Installation
6
+
7
+ ```bash
8
+ npm install
9
+ ```
10
+
11
+ ## Usage
12
+
13
+ The client provides commands to submit benchmarks, check their status, and view results.
14
+
15
+ From the client directory:
16
+ ```bash
17
+ npm run cli -- <command> [options]
18
+ ```
19
+
20
+ Or from the root directory:
21
+ ```bash
22
+ npm run cli --prefix client -- <command> [options]
23
+ ```
24
+
25
+ ### Submit a Benchmark
26
+
27
+ ```bash
28
+ npm run cli -- submit <modelId> <task> [options]
29
+ ```
30
+
31
+ **Options:**
32
+ - `--platform <node|web>` - Platform to run on (default: node)
33
+ - `--mode <warm|cold>` - Cache mode (default: warm)
34
+ - `--repeats <n>` - Number of times to repeat (default: 3)
35
+ - `--batch-size <n>` - Batch size for inference (default: 1)
36
+ - `--dtype <type>` - Data type (fp32, fp16, q8, etc.)
37
+ - `--device <device>` - Device for web platform (default: webgpu)
38
+ - `--browser <chromium|firefox|webkit>` - Browser for web (default: chromium)
39
+ - `--headed` - Run browser in headed mode
40
+ - `--wait` - Wait for completion
41
+
42
+ **Examples:**
43
+
44
+ ```bash
45
+ # Submit a Node.js benchmark
46
+ npm run cli -- submit Xenova/all-MiniLM-L6-v2 feature-extraction --platform node --repeats 5 --wait
47
+
48
+ # Submit a web benchmark with WebGPU
49
+ npm run cli -- submit Xenova/distilbert-base-uncased fill-mask --platform web --device webgpu --headed
50
+
51
+ # Submit with specific dtype
52
+ npm run cli -- submit Xenova/all-MiniLM-L6-v2 feature-extraction --dtype fp16 --batch-size 4
53
+ ```
54
+
55
+ ### Get Benchmark Result
56
+
57
+ ```bash
58
+ npm run cli -- get <benchmark-id>
59
+ ```
60
+
61
+ ### List All Benchmarks
62
+
63
+ ```bash
64
+ npm run cli -- list
65
+ ```
66
+
67
+ ### Check Queue Status
68
+
69
+ ```bash
70
+ npm run cli -- queue
71
+ ```
72
+
73
+ ## Configuration
74
+
75
+ Set the server URL via environment variable:
76
+
77
+ ```bash
78
+ export BENCH_SERVER_URL=http://localhost:3000
79
+ ```
80
+
81
+ Default: `http://localhost:3000`
client/package-lock.json ADDED
@@ -0,0 +1,770 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "client",
3
+ "version": "0.0.1",
4
+ "lockfileVersion": 3,
5
+ "requires": true,
6
+ "packages": {
7
+ "": {
8
+ "name": "client",
9
+ "version": "0.0.1",
10
+ "dependencies": {
11
+ "yargs": "^17.7.2"
12
+ },
13
+ "devDependencies": {
14
+ "@types/yargs": "^17.0.33",
15
+ "tsx": "^4.20.6",
16
+ "typescript": "^5.9.3"
17
+ }
18
+ },
19
+ "node_modules/@esbuild/aix-ppc64": {
20
+ "version": "0.25.10",
21
+ "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.10.tgz",
22
+ "integrity": "sha512-0NFWnA+7l41irNuaSVlLfgNT12caWJVLzp5eAVhZ0z1qpxbockccEt3s+149rE64VUI3Ml2zt8Nv5JVc4QXTsw==",
23
+ "cpu": [
24
+ "ppc64"
25
+ ],
26
+ "dev": true,
27
+ "license": "MIT",
28
+ "optional": true,
29
+ "os": [
30
+ "aix"
31
+ ],
32
+ "engines": {
33
+ "node": ">=18"
34
+ }
35
+ },
36
+ "node_modules/@esbuild/android-arm": {
37
+ "version": "0.25.10",
38
+ "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.10.tgz",
39
+ "integrity": "sha512-dQAxF1dW1C3zpeCDc5KqIYuZ1tgAdRXNoZP7vkBIRtKZPYe2xVr/d3SkirklCHudW1B45tGiUlz2pUWDfbDD4w==",
40
+ "cpu": [
41
+ "arm"
42
+ ],
43
+ "dev": true,
44
+ "license": "MIT",
45
+ "optional": true,
46
+ "os": [
47
+ "android"
48
+ ],
49
+ "engines": {
50
+ "node": ">=18"
51
+ }
52
+ },
53
+ "node_modules/@esbuild/android-arm64": {
54
+ "version": "0.25.10",
55
+ "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.10.tgz",
56
+ "integrity": "sha512-LSQa7eDahypv/VO6WKohZGPSJDq5OVOo3UoFR1E4t4Gj1W7zEQMUhI+lo81H+DtB+kP+tDgBp+M4oNCwp6kffg==",
57
+ "cpu": [
58
+ "arm64"
59
+ ],
60
+ "dev": true,
61
+ "license": "MIT",
62
+ "optional": true,
63
+ "os": [
64
+ "android"
65
+ ],
66
+ "engines": {
67
+ "node": ">=18"
68
+ }
69
+ },
70
+ "node_modules/@esbuild/android-x64": {
71
+ "version": "0.25.10",
72
+ "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.10.tgz",
73
+ "integrity": "sha512-MiC9CWdPrfhibcXwr39p9ha1x0lZJ9KaVfvzA0Wxwz9ETX4v5CHfF09bx935nHlhi+MxhA63dKRRQLiVgSUtEg==",
74
+ "cpu": [
75
+ "x64"
76
+ ],
77
+ "dev": true,
78
+ "license": "MIT",
79
+ "optional": true,
80
+ "os": [
81
+ "android"
82
+ ],
83
+ "engines": {
84
+ "node": ">=18"
85
+ }
86
+ },
87
+ "node_modules/@esbuild/darwin-arm64": {
88
+ "version": "0.25.10",
89
+ "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.10.tgz",
90
+ "integrity": "sha512-JC74bdXcQEpW9KkV326WpZZjLguSZ3DfS8wrrvPMHgQOIEIG/sPXEN/V8IssoJhbefLRcRqw6RQH2NnpdprtMA==",
91
+ "cpu": [
92
+ "arm64"
93
+ ],
94
+ "dev": true,
95
+ "license": "MIT",
96
+ "optional": true,
97
+ "os": [
98
+ "darwin"
99
+ ],
100
+ "engines": {
101
+ "node": ">=18"
102
+ }
103
+ },
104
+ "node_modules/@esbuild/darwin-x64": {
105
+ "version": "0.25.10",
106
+ "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.10.tgz",
107
+ "integrity": "sha512-tguWg1olF6DGqzws97pKZ8G2L7Ig1vjDmGTwcTuYHbuU6TTjJe5FXbgs5C1BBzHbJ2bo1m3WkQDbWO2PvamRcg==",
108
+ "cpu": [
109
+ "x64"
110
+ ],
111
+ "dev": true,
112
+ "license": "MIT",
113
+ "optional": true,
114
+ "os": [
115
+ "darwin"
116
+ ],
117
+ "engines": {
118
+ "node": ">=18"
119
+ }
120
+ },
121
+ "node_modules/@esbuild/freebsd-arm64": {
122
+ "version": "0.25.10",
123
+ "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.10.tgz",
124
+ "integrity": "sha512-3ZioSQSg1HT2N05YxeJWYR+Libe3bREVSdWhEEgExWaDtyFbbXWb49QgPvFH8u03vUPX10JhJPcz7s9t9+boWg==",
125
+ "cpu": [
126
+ "arm64"
127
+ ],
128
+ "dev": true,
129
+ "license": "MIT",
130
+ "optional": true,
131
+ "os": [
132
+ "freebsd"
133
+ ],
134
+ "engines": {
135
+ "node": ">=18"
136
+ }
137
+ },
138
+ "node_modules/@esbuild/freebsd-x64": {
139
+ "version": "0.25.10",
140
+ "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.10.tgz",
141
+ "integrity": "sha512-LLgJfHJk014Aa4anGDbh8bmI5Lk+QidDmGzuC2D+vP7mv/GeSN+H39zOf7pN5N8p059FcOfs2bVlrRr4SK9WxA==",
142
+ "cpu": [
143
+ "x64"
144
+ ],
145
+ "dev": true,
146
+ "license": "MIT",
147
+ "optional": true,
148
+ "os": [
149
+ "freebsd"
150
+ ],
151
+ "engines": {
152
+ "node": ">=18"
153
+ }
154
+ },
155
+ "node_modules/@esbuild/linux-arm": {
156
+ "version": "0.25.10",
157
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.10.tgz",
158
+ "integrity": "sha512-oR31GtBTFYCqEBALI9r6WxoU/ZofZl962pouZRTEYECvNF/dtXKku8YXcJkhgK/beU+zedXfIzHijSRapJY3vg==",
159
+ "cpu": [
160
+ "arm"
161
+ ],
162
+ "dev": true,
163
+ "license": "MIT",
164
+ "optional": true,
165
+ "os": [
166
+ "linux"
167
+ ],
168
+ "engines": {
169
+ "node": ">=18"
170
+ }
171
+ },
172
+ "node_modules/@esbuild/linux-arm64": {
173
+ "version": "0.25.10",
174
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.10.tgz",
175
+ "integrity": "sha512-5luJWN6YKBsawd5f9i4+c+geYiVEw20FVW5x0v1kEMWNq8UctFjDiMATBxLvmmHA4bf7F6hTRaJgtghFr9iziQ==",
176
+ "cpu": [
177
+ "arm64"
178
+ ],
179
+ "dev": true,
180
+ "license": "MIT",
181
+ "optional": true,
182
+ "os": [
183
+ "linux"
184
+ ],
185
+ "engines": {
186
+ "node": ">=18"
187
+ }
188
+ },
189
+ "node_modules/@esbuild/linux-ia32": {
190
+ "version": "0.25.10",
191
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.10.tgz",
192
+ "integrity": "sha512-NrSCx2Kim3EnnWgS4Txn0QGt0Xipoumb6z6sUtl5bOEZIVKhzfyp/Lyw4C1DIYvzeW/5mWYPBFJU3a/8Yr75DQ==",
193
+ "cpu": [
194
+ "ia32"
195
+ ],
196
+ "dev": true,
197
+ "license": "MIT",
198
+ "optional": true,
199
+ "os": [
200
+ "linux"
201
+ ],
202
+ "engines": {
203
+ "node": ">=18"
204
+ }
205
+ },
206
+ "node_modules/@esbuild/linux-loong64": {
207
+ "version": "0.25.10",
208
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.10.tgz",
209
+ "integrity": "sha512-xoSphrd4AZda8+rUDDfD9J6FUMjrkTz8itpTITM4/xgerAZZcFW7Dv+sun7333IfKxGG8gAq+3NbfEMJfiY+Eg==",
210
+ "cpu": [
211
+ "loong64"
212
+ ],
213
+ "dev": true,
214
+ "license": "MIT",
215
+ "optional": true,
216
+ "os": [
217
+ "linux"
218
+ ],
219
+ "engines": {
220
+ "node": ">=18"
221
+ }
222
+ },
223
+ "node_modules/@esbuild/linux-mips64el": {
224
+ "version": "0.25.10",
225
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.10.tgz",
226
+ "integrity": "sha512-ab6eiuCwoMmYDyTnyptoKkVS3k8fy/1Uvq7Dj5czXI6DF2GqD2ToInBI0SHOp5/X1BdZ26RKc5+qjQNGRBelRA==",
227
+ "cpu": [
228
+ "mips64el"
229
+ ],
230
+ "dev": true,
231
+ "license": "MIT",
232
+ "optional": true,
233
+ "os": [
234
+ "linux"
235
+ ],
236
+ "engines": {
237
+ "node": ">=18"
238
+ }
239
+ },
240
+ "node_modules/@esbuild/linux-ppc64": {
241
+ "version": "0.25.10",
242
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.10.tgz",
243
+ "integrity": "sha512-NLinzzOgZQsGpsTkEbdJTCanwA5/wozN9dSgEl12haXJBzMTpssebuXR42bthOF3z7zXFWH1AmvWunUCkBE4EA==",
244
+ "cpu": [
245
+ "ppc64"
246
+ ],
247
+ "dev": true,
248
+ "license": "MIT",
249
+ "optional": true,
250
+ "os": [
251
+ "linux"
252
+ ],
253
+ "engines": {
254
+ "node": ">=18"
255
+ }
256
+ },
257
+ "node_modules/@esbuild/linux-riscv64": {
258
+ "version": "0.25.10",
259
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.10.tgz",
260
+ "integrity": "sha512-FE557XdZDrtX8NMIeA8LBJX3dC2M8VGXwfrQWU7LB5SLOajfJIxmSdyL/gU1m64Zs9CBKvm4UAuBp5aJ8OgnrA==",
261
+ "cpu": [
262
+ "riscv64"
263
+ ],
264
+ "dev": true,
265
+ "license": "MIT",
266
+ "optional": true,
267
+ "os": [
268
+ "linux"
269
+ ],
270
+ "engines": {
271
+ "node": ">=18"
272
+ }
273
+ },
274
+ "node_modules/@esbuild/linux-s390x": {
275
+ "version": "0.25.10",
276
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.10.tgz",
277
+ "integrity": "sha512-3BBSbgzuB9ajLoVZk0mGu+EHlBwkusRmeNYdqmznmMc9zGASFjSsxgkNsqmXugpPk00gJ0JNKh/97nxmjctdew==",
278
+ "cpu": [
279
+ "s390x"
280
+ ],
281
+ "dev": true,
282
+ "license": "MIT",
283
+ "optional": true,
284
+ "os": [
285
+ "linux"
286
+ ],
287
+ "engines": {
288
+ "node": ">=18"
289
+ }
290
+ },
291
+ "node_modules/@esbuild/linux-x64": {
292
+ "version": "0.25.10",
293
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.10.tgz",
294
+ "integrity": "sha512-QSX81KhFoZGwenVyPoberggdW1nrQZSvfVDAIUXr3WqLRZGZqWk/P4T8p2SP+de2Sr5HPcvjhcJzEiulKgnxtA==",
295
+ "cpu": [
296
+ "x64"
297
+ ],
298
+ "dev": true,
299
+ "license": "MIT",
300
+ "optional": true,
301
+ "os": [
302
+ "linux"
303
+ ],
304
+ "engines": {
305
+ "node": ">=18"
306
+ }
307
+ },
308
+ "node_modules/@esbuild/netbsd-arm64": {
309
+ "version": "0.25.10",
310
+ "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.10.tgz",
311
+ "integrity": "sha512-AKQM3gfYfSW8XRk8DdMCzaLUFB15dTrZfnX8WXQoOUpUBQ+NaAFCP1kPS/ykbbGYz7rxn0WS48/81l9hFl3u4A==",
312
+ "cpu": [
313
+ "arm64"
314
+ ],
315
+ "dev": true,
316
+ "license": "MIT",
317
+ "optional": true,
318
+ "os": [
319
+ "netbsd"
320
+ ],
321
+ "engines": {
322
+ "node": ">=18"
323
+ }
324
+ },
325
+ "node_modules/@esbuild/netbsd-x64": {
326
+ "version": "0.25.10",
327
+ "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.10.tgz",
328
+ "integrity": "sha512-7RTytDPGU6fek/hWuN9qQpeGPBZFfB4zZgcz2VK2Z5VpdUxEI8JKYsg3JfO0n/Z1E/6l05n0unDCNc4HnhQGig==",
329
+ "cpu": [
330
+ "x64"
331
+ ],
332
+ "dev": true,
333
+ "license": "MIT",
334
+ "optional": true,
335
+ "os": [
336
+ "netbsd"
337
+ ],
338
+ "engines": {
339
+ "node": ">=18"
340
+ }
341
+ },
342
+ "node_modules/@esbuild/openbsd-arm64": {
343
+ "version": "0.25.10",
344
+ "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.10.tgz",
345
+ "integrity": "sha512-5Se0VM9Wtq797YFn+dLimf2Zx6McttsH2olUBsDml+lm0GOCRVebRWUvDtkY4BWYv/3NgzS8b/UM3jQNh5hYyw==",
346
+ "cpu": [
347
+ "arm64"
348
+ ],
349
+ "dev": true,
350
+ "license": "MIT",
351
+ "optional": true,
352
+ "os": [
353
+ "openbsd"
354
+ ],
355
+ "engines": {
356
+ "node": ">=18"
357
+ }
358
+ },
359
+ "node_modules/@esbuild/openbsd-x64": {
360
+ "version": "0.25.10",
361
+ "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.10.tgz",
362
+ "integrity": "sha512-XkA4frq1TLj4bEMB+2HnI0+4RnjbuGZfet2gs/LNs5Hc7D89ZQBHQ0gL2ND6Lzu1+QVkjp3x1gIcPKzRNP8bXw==",
363
+ "cpu": [
364
+ "x64"
365
+ ],
366
+ "dev": true,
367
+ "license": "MIT",
368
+ "optional": true,
369
+ "os": [
370
+ "openbsd"
371
+ ],
372
+ "engines": {
373
+ "node": ">=18"
374
+ }
375
+ },
376
+ "node_modules/@esbuild/openharmony-arm64": {
377
+ "version": "0.25.10",
378
+ "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.10.tgz",
379
+ "integrity": "sha512-AVTSBhTX8Y/Fz6OmIVBip9tJzZEUcY8WLh7I59+upa5/GPhh2/aM6bvOMQySspnCCHvFi79kMtdJS1w0DXAeag==",
380
+ "cpu": [
381
+ "arm64"
382
+ ],
383
+ "dev": true,
384
+ "license": "MIT",
385
+ "optional": true,
386
+ "os": [
387
+ "openharmony"
388
+ ],
389
+ "engines": {
390
+ "node": ">=18"
391
+ }
392
+ },
393
+ "node_modules/@esbuild/sunos-x64": {
394
+ "version": "0.25.10",
395
+ "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.10.tgz",
396
+ "integrity": "sha512-fswk3XT0Uf2pGJmOpDB7yknqhVkJQkAQOcW/ccVOtfx05LkbWOaRAtn5SaqXypeKQra1QaEa841PgrSL9ubSPQ==",
397
+ "cpu": [
398
+ "x64"
399
+ ],
400
+ "dev": true,
401
+ "license": "MIT",
402
+ "optional": true,
403
+ "os": [
404
+ "sunos"
405
+ ],
406
+ "engines": {
407
+ "node": ">=18"
408
+ }
409
+ },
410
+ "node_modules/@esbuild/win32-arm64": {
411
+ "version": "0.25.10",
412
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.10.tgz",
413
+ "integrity": "sha512-ah+9b59KDTSfpaCg6VdJoOQvKjI33nTaQr4UluQwW7aEwZQsbMCfTmfEO4VyewOxx4RaDT/xCy9ra2GPWmO7Kw==",
414
+ "cpu": [
415
+ "arm64"
416
+ ],
417
+ "dev": true,
418
+ "license": "MIT",
419
+ "optional": true,
420
+ "os": [
421
+ "win32"
422
+ ],
423
+ "engines": {
424
+ "node": ">=18"
425
+ }
426
+ },
427
+ "node_modules/@esbuild/win32-ia32": {
428
+ "version": "0.25.10",
429
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.10.tgz",
430
+ "integrity": "sha512-QHPDbKkrGO8/cz9LKVnJU22HOi4pxZnZhhA2HYHez5Pz4JeffhDjf85E57Oyco163GnzNCVkZK0b/n4Y0UHcSw==",
431
+ "cpu": [
432
+ "ia32"
433
+ ],
434
+ "dev": true,
435
+ "license": "MIT",
436
+ "optional": true,
437
+ "os": [
438
+ "win32"
439
+ ],
440
+ "engines": {
441
+ "node": ">=18"
442
+ }
443
+ },
444
+ "node_modules/@esbuild/win32-x64": {
445
+ "version": "0.25.10",
446
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.10.tgz",
447
+ "integrity": "sha512-9KpxSVFCu0iK1owoez6aC/s/EdUQLDN3adTxGCqxMVhrPDj6bt5dbrHDXUuq+Bs2vATFBBrQS5vdQ/Ed2P+nbw==",
448
+ "cpu": [
449
+ "x64"
450
+ ],
451
+ "dev": true,
452
+ "license": "MIT",
453
+ "optional": true,
454
+ "os": [
455
+ "win32"
456
+ ],
457
+ "engines": {
458
+ "node": ">=18"
459
+ }
460
+ },
461
+ "node_modules/@types/yargs": {
462
+ "version": "17.0.33",
463
+ "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz",
464
+ "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==",
465
+ "dev": true,
466
+ "license": "MIT",
467
+ "dependencies": {
468
+ "@types/yargs-parser": "*"
469
+ }
470
+ },
471
+ "node_modules/@types/yargs-parser": {
472
+ "version": "21.0.3",
473
+ "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz",
474
+ "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==",
475
+ "dev": true,
476
+ "license": "MIT"
477
+ },
478
+ "node_modules/ansi-regex": {
479
+ "version": "5.0.1",
480
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
481
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
482
+ "license": "MIT",
483
+ "engines": {
484
+ "node": ">=8"
485
+ }
486
+ },
487
+ "node_modules/ansi-styles": {
488
+ "version": "4.3.0",
489
+ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
490
+ "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
491
+ "license": "MIT",
492
+ "dependencies": {
493
+ "color-convert": "^2.0.1"
494
+ },
495
+ "engines": {
496
+ "node": ">=8"
497
+ },
498
+ "funding": {
499
+ "url": "https://github.com/chalk/ansi-styles?sponsor=1"
500
+ }
501
+ },
502
+ "node_modules/cliui": {
503
+ "version": "8.0.1",
504
+ "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
505
+ "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
506
+ "license": "ISC",
507
+ "dependencies": {
508
+ "string-width": "^4.2.0",
509
+ "strip-ansi": "^6.0.1",
510
+ "wrap-ansi": "^7.0.0"
511
+ },
512
+ "engines": {
513
+ "node": ">=12"
514
+ }
515
+ },
516
+ "node_modules/color-convert": {
517
+ "version": "2.0.1",
518
+ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
519
+ "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
520
+ "license": "MIT",
521
+ "dependencies": {
522
+ "color-name": "~1.1.4"
523
+ },
524
+ "engines": {
525
+ "node": ">=7.0.0"
526
+ }
527
+ },
528
+ "node_modules/color-name": {
529
+ "version": "1.1.4",
530
+ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
531
+ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
532
+ "license": "MIT"
533
+ },
534
+ "node_modules/emoji-regex": {
535
+ "version": "8.0.0",
536
+ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
537
+ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
538
+ "license": "MIT"
539
+ },
540
+ "node_modules/esbuild": {
541
+ "version": "0.25.10",
542
+ "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.10.tgz",
543
+ "integrity": "sha512-9RiGKvCwaqxO2owP61uQ4BgNborAQskMR6QusfWzQqv7AZOg5oGehdY2pRJMTKuwxd1IDBP4rSbI5lHzU7SMsQ==",
544
+ "dev": true,
545
+ "hasInstallScript": true,
546
+ "license": "MIT",
547
+ "bin": {
548
+ "esbuild": "bin/esbuild"
549
+ },
550
+ "engines": {
551
+ "node": ">=18"
552
+ },
553
+ "optionalDependencies": {
554
+ "@esbuild/aix-ppc64": "0.25.10",
555
+ "@esbuild/android-arm": "0.25.10",
556
+ "@esbuild/android-arm64": "0.25.10",
557
+ "@esbuild/android-x64": "0.25.10",
558
+ "@esbuild/darwin-arm64": "0.25.10",
559
+ "@esbuild/darwin-x64": "0.25.10",
560
+ "@esbuild/freebsd-arm64": "0.25.10",
561
+ "@esbuild/freebsd-x64": "0.25.10",
562
+ "@esbuild/linux-arm": "0.25.10",
563
+ "@esbuild/linux-arm64": "0.25.10",
564
+ "@esbuild/linux-ia32": "0.25.10",
565
+ "@esbuild/linux-loong64": "0.25.10",
566
+ "@esbuild/linux-mips64el": "0.25.10",
567
+ "@esbuild/linux-ppc64": "0.25.10",
568
+ "@esbuild/linux-riscv64": "0.25.10",
569
+ "@esbuild/linux-s390x": "0.25.10",
570
+ "@esbuild/linux-x64": "0.25.10",
571
+ "@esbuild/netbsd-arm64": "0.25.10",
572
+ "@esbuild/netbsd-x64": "0.25.10",
573
+ "@esbuild/openbsd-arm64": "0.25.10",
574
+ "@esbuild/openbsd-x64": "0.25.10",
575
+ "@esbuild/openharmony-arm64": "0.25.10",
576
+ "@esbuild/sunos-x64": "0.25.10",
577
+ "@esbuild/win32-arm64": "0.25.10",
578
+ "@esbuild/win32-ia32": "0.25.10",
579
+ "@esbuild/win32-x64": "0.25.10"
580
+ }
581
+ },
582
+ "node_modules/escalade": {
583
+ "version": "3.2.0",
584
+ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
585
+ "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
586
+ "license": "MIT",
587
+ "engines": {
588
+ "node": ">=6"
589
+ }
590
+ },
591
+ "node_modules/fsevents": {
592
+ "version": "2.3.3",
593
+ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
594
+ "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
595
+ "dev": true,
596
+ "hasInstallScript": true,
597
+ "license": "MIT",
598
+ "optional": true,
599
+ "os": [
600
+ "darwin"
601
+ ],
602
+ "engines": {
603
+ "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
604
+ }
605
+ },
606
+ "node_modules/get-caller-file": {
607
+ "version": "2.0.5",
608
+ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
609
+ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
610
+ "license": "ISC",
611
+ "engines": {
612
+ "node": "6.* || 8.* || >= 10.*"
613
+ }
614
+ },
615
+ "node_modules/get-tsconfig": {
616
+ "version": "4.10.1",
617
+ "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz",
618
+ "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==",
619
+ "dev": true,
620
+ "license": "MIT",
621
+ "dependencies": {
622
+ "resolve-pkg-maps": "^1.0.0"
623
+ },
624
+ "funding": {
625
+ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
626
+ }
627
+ },
628
+ "node_modules/is-fullwidth-code-point": {
629
+ "version": "3.0.0",
630
+ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
631
+ "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
632
+ "license": "MIT",
633
+ "engines": {
634
+ "node": ">=8"
635
+ }
636
+ },
637
+ "node_modules/require-directory": {
638
+ "version": "2.1.1",
639
+ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
640
+ "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
641
+ "license": "MIT",
642
+ "engines": {
643
+ "node": ">=0.10.0"
644
+ }
645
+ },
646
+ "node_modules/resolve-pkg-maps": {
647
+ "version": "1.0.0",
648
+ "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
649
+ "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
650
+ "dev": true,
651
+ "license": "MIT",
652
+ "funding": {
653
+ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
654
+ }
655
+ },
656
+ "node_modules/string-width": {
657
+ "version": "4.2.3",
658
+ "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
659
+ "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
660
+ "license": "MIT",
661
+ "dependencies": {
662
+ "emoji-regex": "^8.0.0",
663
+ "is-fullwidth-code-point": "^3.0.0",
664
+ "strip-ansi": "^6.0.1"
665
+ },
666
+ "engines": {
667
+ "node": ">=8"
668
+ }
669
+ },
670
+ "node_modules/strip-ansi": {
671
+ "version": "6.0.1",
672
+ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
673
+ "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
674
+ "license": "MIT",
675
+ "dependencies": {
676
+ "ansi-regex": "^5.0.1"
677
+ },
678
+ "engines": {
679
+ "node": ">=8"
680
+ }
681
+ },
682
+ "node_modules/tsx": {
683
+ "version": "4.20.6",
684
+ "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz",
685
+ "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==",
686
+ "dev": true,
687
+ "license": "MIT",
688
+ "dependencies": {
689
+ "esbuild": "~0.25.0",
690
+ "get-tsconfig": "^4.7.5"
691
+ },
692
+ "bin": {
693
+ "tsx": "dist/cli.mjs"
694
+ },
695
+ "engines": {
696
+ "node": ">=18.0.0"
697
+ },
698
+ "optionalDependencies": {
699
+ "fsevents": "~2.3.3"
700
+ }
701
+ },
702
+ "node_modules/typescript": {
703
+ "version": "5.9.3",
704
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
705
+ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
706
+ "dev": true,
707
+ "license": "Apache-2.0",
708
+ "bin": {
709
+ "tsc": "bin/tsc",
710
+ "tsserver": "bin/tsserver"
711
+ },
712
+ "engines": {
713
+ "node": ">=14.17"
714
+ }
715
+ },
716
+ "node_modules/wrap-ansi": {
717
+ "version": "7.0.0",
718
+ "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
719
+ "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
720
+ "license": "MIT",
721
+ "dependencies": {
722
+ "ansi-styles": "^4.0.0",
723
+ "string-width": "^4.1.0",
724
+ "strip-ansi": "^6.0.0"
725
+ },
726
+ "engines": {
727
+ "node": ">=10"
728
+ },
729
+ "funding": {
730
+ "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
731
+ }
732
+ },
733
+ "node_modules/y18n": {
734
+ "version": "5.0.8",
735
+ "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
736
+ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
737
+ "license": "ISC",
738
+ "engines": {
739
+ "node": ">=10"
740
+ }
741
+ },
742
+ "node_modules/yargs": {
743
+ "version": "17.7.2",
744
+ "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
745
+ "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
746
+ "license": "MIT",
747
+ "dependencies": {
748
+ "cliui": "^8.0.1",
749
+ "escalade": "^3.1.1",
750
+ "get-caller-file": "^2.0.5",
751
+ "require-directory": "^2.1.1",
752
+ "string-width": "^4.2.3",
753
+ "y18n": "^5.0.5",
754
+ "yargs-parser": "^21.1.1"
755
+ },
756
+ "engines": {
757
+ "node": ">=12"
758
+ }
759
+ },
760
+ "node_modules/yargs-parser": {
761
+ "version": "21.1.1",
762
+ "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
763
+ "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
764
+ "license": "ISC",
765
+ "engines": {
766
+ "node": ">=12"
767
+ }
768
+ }
769
+ }
770
+ }
client/package.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "client",
3
+ "private": true,
4
+ "type": "module",
5
+ "version": "0.0.1",
6
+ "scripts": {
7
+ "cli": "tsx src/index.ts"
8
+ },
9
+ "dependencies": {
10
+ "yargs": "^17.7.2"
11
+ },
12
+ "devDependencies": {
13
+ "@types/yargs": "^17.0.33",
14
+ "tsx": "^4.20.6",
15
+ "typescript": "^5.9.3"
16
+ }
17
+ }
client/src/index.ts ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env node
2
+
3
+ import yargs from "yargs";
4
+ import { hideBin } from "yargs/helpers";
5
+
6
+ const SERVER_URL = process.env.BENCH_SERVER_URL || "http://localhost:3000";
7
+
8
+ interface SubmitOptions {
9
+ platform?: "node" | "web";
10
+ modelId: string;
11
+ task: string;
12
+ mode?: "warm" | "cold";
13
+ repeats?: number;
14
+ dtype?: string;
15
+ batchSize?: number;
16
+ device?: string;
17
+ browser?: string;
18
+ headed?: boolean;
19
+ }
20
+
21
+ async function submitBenchmark(options: SubmitOptions) {
22
+ const response = await fetch(`${SERVER_URL}/api/benchmark`, {
23
+ method: "POST",
24
+ headers: { "Content-Type": "application/json" },
25
+ body: JSON.stringify(options),
26
+ });
27
+
28
+ if (!response.ok) {
29
+ throw new Error(`Failed to submit benchmark: ${response.statusText}`);
30
+ }
31
+
32
+ return await response.json();
33
+ }
34
+
35
+ async function getBenchmark(id: string) {
36
+ const response = await fetch(`${SERVER_URL}/api/benchmark/${id}`);
37
+
38
+ if (!response.ok) {
39
+ throw new Error(`Failed to get benchmark: ${response.statusText}`);
40
+ }
41
+
42
+ return await response.json();
43
+ }
44
+
45
+ async function listBenchmarks() {
46
+ const response = await fetch(`${SERVER_URL}/api/benchmarks`);
47
+
48
+ if (!response.ok) {
49
+ throw new Error(`Failed to list benchmarks: ${response.statusText}`);
50
+ }
51
+
52
+ return await response.json();
53
+ }
54
+
55
+ async function getQueueStatus() {
56
+ const response = await fetch(`${SERVER_URL}/api/queue`);
57
+
58
+ if (!response.ok) {
59
+ throw new Error(`Failed to get queue status: ${response.statusText}`);
60
+ }
61
+
62
+ return await response.json();
63
+ }
64
+
65
+ async function pollBenchmark(id: string, interval = 2000): Promise<any> {
66
+ return new Promise((resolve, reject) => {
67
+ const check = async () => {
68
+ try {
69
+ const result = await getBenchmark(id);
70
+
71
+ if (result.status === "completed") {
72
+ resolve(result);
73
+ } else if (result.status === "failed") {
74
+ reject(new Error(result.error));
75
+ } else {
76
+ console.log(`Status: ${result.status}...`);
77
+ setTimeout(check, interval);
78
+ }
79
+ } catch (error) {
80
+ reject(error);
81
+ }
82
+ };
83
+ check();
84
+ });
85
+ }
86
+
87
+ yargs(hideBin(process.argv))
88
+ .command(
89
+ "submit <modelId> <task>",
90
+ "Submit a new benchmark request",
91
+ (yargs) => {
92
+ return yargs
93
+ .positional("modelId", {
94
+ describe: "Model ID to benchmark",
95
+ type: "string",
96
+ demandOption: true,
97
+ })
98
+ .positional("task", {
99
+ describe: "Task to perform (e.g., feature-extraction, fill-mask)",
100
+ type: "string",
101
+ demandOption: true,
102
+ })
103
+ .option("platform", {
104
+ describe: "Platform to run on",
105
+ choices: ["node", "web"] as const,
106
+ default: "node" as const,
107
+ })
108
+ .option("mode", {
109
+ describe: "Cache mode",
110
+ choices: ["warm", "cold"] as const,
111
+ default: "warm" as const,
112
+ })
113
+ .option("repeats", {
114
+ describe: "Number of times to repeat the benchmark",
115
+ type: "number",
116
+ default: 3,
117
+ })
118
+ .option("batch-size", {
119
+ describe: "Batch size for inference",
120
+ type: "number",
121
+ default: 1,
122
+ })
123
+ .option("dtype", {
124
+ describe: "Data type (fp32, fp16, q8, etc.)",
125
+ type: "string",
126
+ })
127
+ .option("device", {
128
+ describe: "Device for web platform",
129
+ type: "string",
130
+ default: "webgpu",
131
+ })
132
+ .option("browser", {
133
+ describe: "Browser for web platform",
134
+ choices: ["chromium", "firefox", "webkit"] as const,
135
+ default: "chromium" as const,
136
+ })
137
+ .option("headed", {
138
+ describe: "Run browser in headed mode",
139
+ type: "boolean",
140
+ default: false,
141
+ })
142
+ .option("wait", {
143
+ describe: "Wait for benchmark completion",
144
+ type: "boolean",
145
+ default: false,
146
+ });
147
+ },
148
+ async (argv) => {
149
+ const options: SubmitOptions = {
150
+ modelId: argv.modelId,
151
+ task: argv.task,
152
+ platform: argv.platform,
153
+ mode: argv.mode,
154
+ repeats: argv.repeats,
155
+ batchSize: argv.batchSize,
156
+ device: argv.device,
157
+ browser: argv.browser,
158
+ headed: argv.headed,
159
+ };
160
+
161
+ if (argv.dtype) {
162
+ options.dtype = argv.dtype;
163
+ }
164
+
165
+ console.log("Submitting benchmark...");
166
+ const result = await submitBenchmark(options);
167
+ console.log(`✓ Benchmark queued: ${result.id}`);
168
+ console.log(` Position in queue: ${result.position}`);
169
+
170
+ if (argv.wait) {
171
+ console.log("\nWaiting for completion...");
172
+ const completed = await pollBenchmark(result.id);
173
+ console.log("\n✅ Benchmark completed!");
174
+ console.log(JSON.stringify(completed.result, null, 2));
175
+ } else {
176
+ console.log(`\nCheck status with: bench-client get ${result.id}`);
177
+ }
178
+ }
179
+ )
180
+ .command(
181
+ "get <id>",
182
+ "Get benchmark result by ID",
183
+ (yargs) => {
184
+ return yargs.positional("id", {
185
+ describe: "Benchmark ID",
186
+ type: "string",
187
+ demandOption: true,
188
+ });
189
+ },
190
+ async (argv) => {
191
+ const result = await getBenchmark(argv.id);
192
+ console.log(JSON.stringify(result, null, 2));
193
+ }
194
+ )
195
+ .command(
196
+ "list",
197
+ "List all benchmark results",
198
+ () => {},
199
+ async () => {
200
+ const result = await listBenchmarks();
201
+ console.log(`Total benchmarks: ${result.total}\n`);
202
+ result.results.forEach((b: any) => {
203
+ console.log(`${b.id} - ${b.status} - ${b.platform}/${b.modelId}/${b.task}`);
204
+ });
205
+ }
206
+ )
207
+ .command(
208
+ "queue",
209
+ "Show queue status",
210
+ () => {},
211
+ async () => {
212
+ const result = await getQueueStatus();
213
+ console.log("Queue Status:");
214
+ console.log(` Pending: ${result.status.pending}`);
215
+ console.log(` Running: ${result.status.running}`);
216
+ console.log(` Completed: ${result.status.completed}`);
217
+ console.log(` Failed: ${result.status.failed}`);
218
+
219
+ if (result.queue.length > 0) {
220
+ console.log("\nCurrent Queue:");
221
+ result.queue.forEach((b: any) => {
222
+ console.log(` [${b.status}] ${b.id} - ${b.platform}/${b.modelId}`);
223
+ });
224
+ }
225
+ }
226
+ )
227
+ .demandCommand(1, "You need to specify a command")
228
+ .help()
229
+ .alias("h", "help")
230
+ .strict()
231
+ .parse();
232
+
233
+ export { submitBenchmark, getBenchmark, listBenchmarks, getQueueStatus, pollBenchmark };
client/tsconfig.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "compilerOptions": {
3
+ "target": "ES2022",
4
+ "module": "ES2022",
5
+ "lib": ["ES2022"],
6
+ "moduleResolution": "bundler",
7
+ "strict": true,
8
+ "esModuleInterop": true,
9
+ "skipLibCheck": true,
10
+ "forceConsistentCasingInFileNames": true,
11
+ "resolveJsonModule": true,
12
+ "outDir": "./dist"
13
+ },
14
+ "include": ["src/**/*"]
15
+ }