Repository: promptfoo/promptfoo
Pull request #3323: feat: add model-scan command
Author: login: typpo
Created: 2025-03-08T16:23:25Z | Merged: 2025-03-09T03:44:45Z



Diff:
diff --git a/site/docs/model-audit/index.md b/site/docs/model-audit/index.md
index 73a9205f6d6b..6bf02ab2705d 100644
--- a/site/docs/model-audit/index.md
+++ b/site/docs/model-audit/index.md
@@ -1,5 +1,5 @@
 ---
-sidebar_label: Intro
+sidebar_label: Overview
 sidebar_position: 1
 ---
 
diff --git a/site/sidebars.js b/site/sidebars.js
index bb8d65b305b0..7ca6fadcca92 100644
--- a/site/sidebars.js
+++ b/site/sidebars.js
@@ -267,7 +267,7 @@ const sidebars = {
     },
     {
       type: 'category',
-      label: 'Other Tools',
+      label: 'Tools',
       collapsed: true,
       items: [
         {
diff --git a/src/commands/modelScan.ts b/src/commands/modelScan.ts
new file mode 100644
index 000000000000..f1acec487385
--- /dev/null
+++ b/src/commands/modelScan.ts
@@ -0,0 +1,109 @@
+import chalk from 'chalk';
+import { spawn } from 'child_process';
+import { exec } from 'child_process';
+import type { Command } from 'commander';
+import { promisify } from 'util';
+import logger from '../logger';
+
+const execAsync = promisify(exec);
+
+async function checkModelAuditInstalled(): Promise<boolean> {
+  try {
+    await execAsync('python -c "import modelaudit"');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function modelScanCommand(program: Command): void {
+  program
+    .command('scan-model')
+    .description('Scan ML models for security vulnerabilities')
+    .argument('[paths...]', 'Paths to model files or directories to scan')
+    .option(
+      '-b, --blacklist <pattern>',
+      'Additional blacklist patterns to check against model names',
+      (val: string, acc: string[]) => [...acc, val],
+      [] as string[],
+    )
+    .option('-f, --format <format>', 'Output format (text or json)', 'text')
+    .option('-o, --output <path>', 'Output file path (prints to stdout if not specified)')
+    .option(
+      '-t, --timeout <seconds>',
+      'Scan timeout in seconds',
+      (val) => Number.parseInt(val, 10),
+      300,
+    )
+    .option('-v, --verbose', 'Enable verbose output')
+    .option('--max-file-size <bytes>', 'Maximum file size to scan in bytes')
+    .action(async (paths: string[], options) => {
+      if (!paths || paths.length === 0) {
+        logger.error(
+          'No paths specified. Please provide at least one model file or directory to scan.',
+        );
+        process.exit(1);
+      }
+
+      // Check if modelaudit is installed
+      const isModelAuditInstalled = await checkModelAuditInstalled();
+      if (!isModelAuditInstalled) {
+        logger.error('ModelAudit is not installed.');
+        logger.info(`Please install it using: ${chalk.green('pip install modelaudit')}`);
+        logger.info('For more information, visit: https://www.promptfoo.dev/docs/model-audit/');
+        process.exit(1);
+      }
+
+      const args = ['-m', 'modelaudit'];
+
+      // Add all paths
+      args.push(...paths);
+
+      // Add options
+      if (options.blacklist && options.blacklist.length > 0) {
+        options.blacklist.forEach((pattern: string) => {
+          args.push('--blacklist', pattern);
+        });
+      }
+
+      if (options.format) {
+        args.push('--format', options.format);
+      }
+
+      if (options.output) {
+        args.push('--output', options.output);
+      }
+
+      if (options.timeout) {
+        args.push('--timeout', options.timeout);
+      }
+
+      if (options.verbose) {
+        args.push('--verbose');
+      }
+
+      if (options.maxFileSize) {
+        args.push('--max-file-size', options.maxFileSize);
+      }
+
+      logger.info(`Running model scan on: ${paths.join(', ')}`);
+
+      const modelAudit = spawn('python', args, { stdio: 'inherit' });
+
+      modelAudit.on('error', (error) => {
+        logger.error(`Failed to start modelaudit: ${error.message}`);
+        logger.info('Make sure modelaudit is installed and available in your PATH.');
+        logger.info('Install it using: pip install modelaudit');
+        process.exit(1);
+      });
+
+      modelAudit.on('close', (code) => {
+        if (code === 0) {
+          logger.info('Model scan completed successfully.');
+        } else {
+          logger.error(`Model scan completed with issues. Exit code: ${code}`);
+          process.exit(code || 1);
+        }
+      });
+    });
+}
diff --git a/src/main.ts b/src/main.ts
index 38f5c1a82f5c..232ff4c578c3 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -14,6 +14,7 @@ import { generateDatasetCommand } from './commands/generate/dataset';
 import { importCommand } from './commands/import';
 import { initCommand } from './commands/init';
 import { listCommand } from './commands/list';
+import { modelScanCommand } from './commands/modelScan';
 import { shareCommand } from './commands/share';
 import { showCommand } from './commands/show';
 import { viewCommand } from './commands/view';
@@ -59,10 +60,11 @@ async function main() {
   debugCommand(program, defaultConfig, defaultConfigPath);
   deleteCommand(program);
   exportCommand(program);
-  feedbackCommand(program);
   const generateCommand = program.command('generate').description('Generate synthetic data');
+  feedbackCommand(program);
   importCommand(program);
   listCommand(program);
+  modelScanCommand(program);
   showCommand(program);
 
   generateDatasetCommand(generateCommand, defaultConfig, defaultConfigPath);


Reviews:
user: login: ellipsis-dev[bot]
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-08T16:26:02Z
user: login: mldangelo
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-08T17:05:20Z
user: login: mldangelo
state: DISMISSED
author_association: MEMBER
submitted_at: 2025-03-08T17:06:35Z
user: login: ellipsis-dev[bot]
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-09T01:24:33Z
user: login: ellipsis-dev[bot]
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-09T03:18:30Z

Review comments:
diff_hunk: @@ -0,0 +1,103 @@
+import { spawn } from 'child_process';
+import { exec } from 'child_process';
+import type { Command } from 'commander';
+import { promisify } from 'util';
+import logger from '../logger';
+
+const execAsync = promisify(exec);
+
+async function checkModelAuditInstalled(): Promise<boolean> {
+  try {
+    await execAsync('python -c "import modelaudit"');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function modelScanCommand(program: Command): void {
+  program
+    .command('scan-model')
+    .description('Scan ML models for security vulnerabilities')
+    .argument('[paths...]', 'Paths to model files or directories to scan')
+    .option(
+      '-b, --blacklist <pattern>',
+      'Additional blacklist patterns to check against model names',
+      (val: string, acc: string[]) => [...acc, val],
+      [] as string[],
+    )
+    .option('-f, --format <format>', 'Output format (text or json)', 'text')
+    .option('-o, --output <path>', 'Output file path (prints to stdout if not specified)')
+    .option('-t, --timeout <seconds>', 'Scan timeout in seconds', '300')
+    .option('-v, --verbose', 'Enable verbose output')
+    .option('--max-file-size <bytes>', 'Maximum file size to scan in bytes')
+    .action(async (paths: string[], options) => {
+      if (!paths || paths.length === 0) {
+        logger.error(
+          'No paths specified. Please provide at least one model file or directory to scan.',
+        );
+        process.exit(1);
+      }
+
+      // Check if modelaudit is installed
+      const isModelAuditInstalled = await checkModelAuditInstalled();
+      if (!isModelAuditInstalled) {
+        logger.error('ModelAudit is not installed.');
+        logger.info('Please install it using: pip install modelaudit');
+        logger.info('For more information, visit: https://www.promptfoo.dev/docs/model-audit/');
+        process.exit(1);
+      }
+
+      const args = ['modelaudit'];
+
+      // Add all paths
+      args.push(...paths);
+
+      // Add options
+      if (options.blacklist && options.blacklist.length > 0) {
+        options.blacklist.forEach((pattern: string) => {
+          args.push('--blacklist', pattern);
+        });
+      }
+
+      if (options.format) {
+        args.push('--format', options.format);
+      }
+
+      if (options.output) {
+        args.push('--output', options.output);
+      }
+
+      if (options.timeout) {
+        args.push('--timeout', options.timeout);
+      }
+
+      if (options.verbose) {
+        args.push('--verbose');
+      }
+
+      if (options.maxFileSize) {
+        args.push('--max-file-size', options.maxFileSize);
+      }
+
+      logger.info(`Running model scan on: ${paths.join(', ')}`);
+
+      const modelAudit = spawn('python', args, { stdio: 'inherit' });
path: src/commands/modelScan.ts
user: login: ellipsis-dev[bot]
body: Consider using the '-m' flag when invoking modelaudit. In `checkModelAuditInstalled`, it uses `python -c "import modelaudit"`, but here it's spawned as `python modelaudit ...`. Using `python -m modelaudit` would be more consistent.
```suggestion
      const modelAudit = spawn('python', ['-m', ...args], { stdio: 'inherit' });
```
created_at: 2025-03-08T16:26:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,103 @@
+import { spawn } from 'child_process';
+import { exec } from 'child_process';
+import type { Command } from 'commander';
+import { promisify } from 'util';
+import logger from '../logger';
+
+const execAsync = promisify(exec);
+
+async function checkModelAuditInstalled(): Promise<boolean> {
+  try {
+    await execAsync('python -c "import modelaudit"');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function modelScanCommand(program: Command): void {
+  program
+    .command('scan-model')
+    .description('Scan ML models for security vulnerabilities')
+    .argument('[paths...]', 'Paths to model files or directories to scan')
+    .option(
+      '-b, --blacklist <pattern>',
+      'Additional blacklist patterns to check against model names',
+      (val: string, acc: string[]) => [...acc, val],
+      [] as string[],
+    )
+    .option('-f, --format <format>', 'Output format (text or json)', 'text')
+    .option('-o, --output <path>', 'Output file path (prints to stdout if not specified)')
+    .option('-t, --timeout <seconds>', 'Scan timeout in seconds', '300')
path: src/commands/modelScan.ts
user: login: ellipsis-dev[bot]
body: Consider adding parsers for numeric options (e.g. timeout and max-file-size) to validate and convert input to numbers.
```suggestion
    .option('-t, --timeout <seconds>', 'Scan timeout in seconds', parseInt, 300)
```
created_at: 2025-03-08T16:26:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -60,11 +61,12 @@ async function main() {
   deleteCommand(program);
   exportCommand(program);
   feedbackCommand(program);
-  const generateCommand = program.command('generate').description('Generate synthetic data');
path: src/main.ts
user: login: mldangelo
body: just an fyi - this changes the help text display alphabetic order
created_at: 2025-03-08T17:05:19Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,103 @@
+import { spawn } from 'child_process';
+import { exec } from 'child_process';
+import type { Command } from 'commander';
+import { promisify } from 'util';
+import logger from '../logger';
+
+const execAsync = promisify(exec);
+
+async function checkModelAuditInstalled(): Promise<boolean> {
+  try {
+    await execAsync('python -c "import modelaudit"');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function modelScanCommand(program: Command): void {
+  program
+    .command('scan-model')
+    .description('Scan ML models for security vulnerabilities')
+    .argument('[paths...]', 'Paths to model files or directories to scan')
+    .option(
+      '-b, --blacklist <pattern>',
+      'Additional blacklist patterns to check against model names',
+      (val: string, acc: string[]) => [...acc, val],
+      [] as string[],
+    )
+    .option('-f, --format <format>', 'Output format (text or json)', 'text')
+    .option('-o, --output <path>', 'Output file path (prints to stdout if not specified)')
+    .option('-t, --timeout <seconds>', 'Scan timeout in seconds', parseInt, 300)
+    .option('-v, --verbose', 'Enable verbose output')
+    .option('--max-file-size <bytes>', 'Maximum file size to scan in bytes')
+    .action(async (paths: string[], options) => {
+      if (!paths || paths.length === 0) {
+        logger.error(
+          'No paths specified. Please provide at least one model file or directory to scan.',
+        );
+        process.exit(1);
+      }
+
+      // Check if modelaudit is installed
+      const isModelAuditInstalled = await checkModelAuditInstalled();
+      if (!isModelAuditInstalled) {
+        logger.error('ModelAudit is not installed.');
+        logger.info(`Please install it using: ${chalk.green('pip install modelaudit')}`);
path: src/commands/modelScan.ts
user: login: ellipsis-dev[bot]
body: Missing import: 'chalk' is used but not imported.
created_at: 2025-03-09T01:24:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,104 @@
+import chalk from 'chalk';
+import { spawn } from 'child_process';
+import { exec } from 'child_process';
+import type { Command } from 'commander';
+import { promisify } from 'util';
+import logger from '../logger';
+
+const execAsync = promisify(exec);
+
+async function checkModelAuditInstalled(): Promise<boolean> {
+  try {
+    await execAsync('python -c "import modelaudit"');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function modelScanCommand(program: Command): void {
+  program
+    .command('scan-model')
+    .description('Scan ML models for security vulnerabilities')
+    .argument('[paths...]', 'Paths to model files or directories to scan')
+    .option(
+      '-b, --blacklist <pattern>',
+      'Additional blacklist patterns to check against model names',
+      (val: string, acc: string[]) => [...acc, val],
+      [] as string[],
+    )
+    .option('-f, --format <format>', 'Output format (text or json)', 'text')
+    .option('-o, --output <path>', 'Output file path (prints to stdout if not specified)')
+    .option('-t, --timeout <seconds>', 'Scan timeout in seconds', Number.parseInt, 300)
path: src/commands/modelScan.ts
user: login: ellipsis-dev[bot]
body: Consider wrapping `Number.parseInt` with an explicit radix (e.g., `(val) => Number.parseInt(val, 10)`) to ensure consistent number parsing.
```suggestion
    .option('-t, --timeout <seconds>', 'Scan timeout in seconds', (val) => Number.parseInt(val, 10), 300)
```
created_at: 2025-03-09T03:18:30Z
author_association: CONTRIBUTOR

PR comments:
user: login: gru-agent[bot]
created_at: 2025-03-08T16:23:41Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/e9e48484-2ed0-4a21-b90a-2763022ce022) | 4dc0582a3dd0b9bf81149d4a2408ace4465a2a6b | 🚫 Skipped | No files need to be tested {"src/commands/modelScan.ts":"target file(src/commands/modelScan.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","src/main.ts":"target file(src/main.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js"} |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR
user: login: gru-agent[bot]
created_at: 2025-03-08T16:23:46Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/96fe90ff-aae2-4afe-97d5-993604eeb712) | 4dc0582a3dd0b9bf81149d4a2408ace4465a2a6b | ✅ Finished | Cancelled by auto rebase |

### Files

| File | Pull Request |
| ---- | ------------ |
| src/commands/modelScan.ts | 🛑 Cancelled (Cancelled by auto rebase) |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR
user: login: gru-agent[bot]
created_at: 2025-03-08T16:24:38Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/c9d5d4b5-de5e-4bd6-a583-d487e4703d7c) | beea9e0649f47affe6756d32735aa7b7ff9afe04 | 🚫 Skipped | No files need to be tested {"site/docs/model-audit/index.md":"target file(site/docs/model-audit/index.md) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","site/sidebars.js":"target file(site/sidebars.js) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","src/commands/modelScan.ts":"target file(src/commands/modelScan.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","src/main.ts":"target file(src/main.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js"} |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR
user: login: gru-agent[bot]
created_at: 2025-03-08T16:24:41Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/bd5a3564-599b-4a40-88f2-c499c84cf45b) | beea9e0649f47affe6756d32735aa7b7ff9afe04 | ✅ Finished |  |

### Files

| File | Pull Request |
| ---- | ------------ |
| src/commands/modelScan.ts | 🔴 Closed https://github.com/promptfoo/promptfoo/pull/3324 |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR