Repository: promptfoo/promptfoo (ts)

Issue:
Clearly document that the tool sends telemetry, possibly with explicit opt in
First up, thanks for working on this - it's an interesting project.

It's usually a surprise to find that an app of this sort is collecting telemetry, particularly when it's not clearly documented as doing so.

At a minimum I'd suggest that a clear message is output whenever telemetry will be collected with instructions to disable telemetry, or disable the clear message (an explicit opt in).

Pull request:
Add telemetry notice
Closes #35

Patch:
diff --git a/examples/langchain-python/requirements.txt b/examples/langchain-python/requirements.txt
index 7530e0d40..6c0103ef0 100644
--- a/examples/langchain-python/requirements.txt
+++ b/examples/langchain-python/requirements.txt
@@ -8,7 +8,7 @@ dataclasses-json==0.5.9
 frozenlist==1.3.3
 greenlet==2.0.2
 idna==3.4
-langchain==0.0.225
+langchain==0.0.228
 langchainplus-sdk==0.0.20
 marshmallow==3.19.0
 marshmallow-enum==1.5.1
diff --git a/src/index.ts b/src/index.ts
index 9798c8b6f..a9bb7e164 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -26,6 +26,7 @@ async function evaluate(testSuite: EvaluateTestSuite, options: EvaluateOptions =
       display: promptContent,
     })),
   };
+  telemetry.maybeShowNotice();
   const ret = await doEvaluate(constructedTestSuite, options);
   await telemetry.send();
   return ret;
diff --git a/src/main.ts b/src/main.ts
index 043e4da55..0d2578db9 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -56,10 +56,10 @@ function createDummyFiles(directory: string | null) {
   writeFileSync(pathJoin(process.cwd(), directory, 'README.md'), DEFAULT_README);
 
   if (directory === '.') {
-    logger.info('Wrote prompts.txt and promptfooconfig.yaml. Open README.md to get started!');
+    logger.info(chalk.green.bold('Wrote prompts.txt and promptfooconfig.yaml. Open README.md to get started!'));
   } else {
-    logger.info(`Wrote prompts.txt and promptfooconfig.yaml to ./${directory}`);
-    logger.info(`\`cd ${directory}\` and open README.md to get started!`);
+    logger.info(chalk.green.bold(`Wrote prompts.txt and promptfooconfig.yaml to ./${directory}`));
+    logger.info(chalk.green(`\`cd ${directory}\` and open README.md to get started!`));
   }
 }
 
@@ -102,6 +102,7 @@ async function main() {
     .command('init [directory]')
     .description('Initialize project with dummy files')
     .action(async (directory: string | null) => {
+      telemetry.maybeShowNotice();
       createDummyFiles(directory);
       telemetry.record('command_used', {
         name: 'init',
@@ -114,6 +115,7 @@ async function main() {
     .description('Start browser ui')
     .option('-p, --port <number>', 'Port number', '15500')
     .action(async (cmdObj: { port: number } & Command) => {
+      telemetry.maybeShowNotice();
       telemetry.record('command_used', {
         name: 'view',
       });
@@ -125,6 +127,7 @@ async function main() {
     .command('share')
     .description('Share your most recent result')
     .action(async (cmdObj: { port: number } & Command) => {
+      telemetry.maybeShowNotice();
       telemetry.record('command_used', {
         name: 'share',
       });
@@ -145,6 +148,7 @@ async function main() {
     .command('clear')
     .description('Clear cache')
     .action(async () => {
+      telemetry.maybeShowNotice();
       await clearCache();
       telemetry.record('command_used', {
         name: 'cache_clear',
@@ -317,6 +321,8 @@ async function main() {
         }
       }
 
+      telemetry.maybeShowNotice();
+
       const border = '='.repeat(process.stdout.columns - 10);
       logger.info(border);
       if (!cmdObj.write) {
@@ -340,13 +346,14 @@ async function main() {
       logger.info(
         `Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`,
       );
-      logger.info('Done.');
 
       telemetry.record('command_used', {
         name: 'eval',
       });
       await telemetry.send();
 
+      logger.info('Done.');
+
       if (cmdObj.view) {
         init(parseInt(cmdObj.view, 10) || 15500);
       }
diff --git a/src/telemetry.ts b/src/telemetry.ts
index ca7f4bd95..34509ea29 100644
--- a/src/telemetry.ts
+++ b/src/telemetry.ts
@@ -1,5 +1,8 @@
+import chalk from 'chalk';
+
 import packageJson from '../package.json';
-import { fetchWithTimeout } from './util';
+import logger from './logger';
+import { fetchWithTimeout, maybeRecordFirstRun } from './util';
 
 type TelemetryEvent = {
   event: string;
@@ -30,6 +33,16 @@ export class Telemetry {
     }
   }
 
+  maybeShowNotice(): void {
+    if (maybeRecordFirstRun()) {
+      logger.info(
+        chalk.gray(
+          'Anonymous telemetry is enabled. For more info, see https://www.promptfoo.dev/docs/configuration/telemetry',
+        ),
+      );
+    }
+  }
+
   async send(): Promise<void> {
     if (!this.disabled && this.events.length > 0) {
       try {
diff --git a/src/util.ts b/src/util.ts
index 103277163..f4d2ab929 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -37,6 +37,46 @@ function parseJson(json: string): any | undefined {
   }
 }
 
+let globalConfigCache: any = null;
+
+export function resetGlobalConfig(): void {
+  globalConfigCache = null;
+}
+
+export function readGlobalConfig(): any {
+  if (!globalConfigCache) {
+    const configDir = getConfigDirectoryPath();
+    const configFilePath = path.join(configDir, 'promptfoo.yaml');
+
+    if (fs.existsSync(configFilePath)) {
+      globalConfigCache = yaml.load(fs.readFileSync(configFilePath, 'utf-8'));
+    } else {
+      if (!fs.existsSync(configDir)) {
+        fs.mkdirSync(configDir, { recursive: true });
+      }
+      globalConfigCache = { hasRun: false };
+      fs.writeFileSync(configFilePath, yaml.dump(globalConfigCache));
+    }
+  }
+
+  return globalConfigCache;
+}
+
+export function maybeRecordFirstRun(): boolean {
+  // Return true if first run
+  try {
+    const config = readGlobalConfig();
+    if (!config.hasRun) {
+      config.hasRun = true;
+      fs.writeFileSync(path.join(getConfigDirectoryPath(), 'promptfoo.yaml'), yaml.dump(config));
+      return true;
+    }
+    return false;
+  } catch (err) {
+    return false;
+  }
+}
+
 export async function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined> {
   if (!fs.existsSync(configPath)) {
     return undefined;


Test patch:
diff --git a/test/telemetry.test.ts b/test/telemetry.test.ts
index c55ad27e1..9b0a67552 100644
--- a/test/telemetry.test.ts
+++ b/test/telemetry.test.ts
@@ -4,6 +4,7 @@ import { fetchWithTimeout } from '../src/util';
 
 jest.mock('../src/util', () => ({
   fetchWithTimeout: jest.fn(),
+  maybeRecordFirstRun: jest.fn(),
 }));
 
 jest.mock('../package.json', () => ({
diff --git a/test/util.test.ts b/test/util.test.ts
index 2ac7cbd90..2b1fc6783 100644
--- a/test/util.test.ts
+++ b/test/util.test.ts
@@ -1,9 +1,18 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
+import yaml from 'js-yaml';
 import { globSync } from 'glob';
 
-import { readVars, readPrompts, writeOutput, readTests } from '../src/util';
+import {
+  readVars,
+  readPrompts,
+  writeOutput,
+  readTests,
+  readGlobalConfig,
+  maybeRecordFirstRun,
+  resetGlobalConfig,
+} from '../src/util';
 
 import type { EvaluateResult, EvaluateTable, Prompt, TestCase } from '../src/types';
 
@@ -17,6 +26,8 @@ jest.mock('fs', () => ({
   writeFileSync: jest.fn(),
   statSync: jest.fn(),
   readdirSync: jest.fn(),
+  existsSync: jest.fn(),
+  mkdirSync: jest.fn(),
 }));
 
 jest.mock('../src/esm.js');
@@ -316,6 +327,64 @@ describe('util', () => {
 
     expect(fs.writeFileSync).toHaveBeenCalledTimes(1);
   });
+
+  describe('readCliConfig', () => {
+    afterEach(() => {
+      jest.clearAllMocks();
+      resetGlobalConfig();
+    });
+
+    test('reads from existing config', () => {
+      const config = { hasRun: false };
+      (fs.existsSync as jest.Mock).mockReturnValue(true);
+      (fs.readFileSync as jest.Mock).mockReturnValue(yaml.dump(config));
+
+      const result = readGlobalConfig();
+
+      expect(fs.existsSync).toHaveBeenCalledTimes(1);
+      expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+      expect(result).toEqual(config);
+    });
+
+    test('creates new config if none exists', () => {
+      (fs.existsSync as jest.Mock).mockReturnValue(false);
+      (fs.writeFileSync as jest.Mock).mockImplementation();
+
+      const result = readGlobalConfig();
+
+      expect(fs.existsSync).toHaveBeenCalledTimes(2);
+      expect(fs.writeFileSync).toHaveBeenCalledTimes(1);
+      expect(result).toEqual({ hasRun: false });
+    });
+  });
+
+  describe('maybeRecordFirstRun', () => {
+    afterEach(() => {
+      resetGlobalConfig();
+      jest.clearAllMocks();
+    });
+
+    test('returns true if it is the first run', () => {
+      (fs.existsSync as jest.Mock).mockReturnValue(false);
+      (fs.writeFileSync as jest.Mock).mockImplementation();
+
+      const result = maybeRecordFirstRun();
+
+      expect(fs.writeFileSync).toHaveBeenCalledTimes(2);
+      expect(result).toBe(true);
+    });
+
+    test('returns false if it is not the first run', () => {
+      const config = { hasRun: true };
+      (fs.existsSync as jest.Mock).mockReturnValue(true);
+      (fs.readFileSync as jest.Mock).mockReturnValue(yaml.dump(config));
+
+      const result = maybeRecordFirstRun();
+
+      expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+      expect(result).toBe(false);
+    });
+  });
 });
 
 describe('readTests', () => {