Repository: promptfoo/promptfoo
Pull request #3273: fix(config): env vars in promptfooconfig.yaml files are strings
Author: login: mldangelo
Created: 2025-03-04T00:46:45Z | Merged: 2025-03-05T10:01:07Z

Fixes the handling of environment variables in the configuration schema by ensuring all values are transformed to strings.

- Updated the schema to transform non-string values to strings.
- Added tests to validate the transformation logic.


Diff:
diff --git a/site/static/config-schema.json b/site/static/config-schema.json
index 5ac541412bb2..0ba5b4a0eefa 100644
--- a/site/static/config-schema.json
+++ b/site/static/config-schema.json
@@ -869,7 +869,17 @@
             {
               "type": "object",
               "additionalProperties": {
-                "type": "string"
+                "anyOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "number"
+                  },
+                  {
+                    "type": "boolean"
+                  }
+                ]
               }
             }
           ]
diff --git a/src/types/index.ts b/src/types/index.ts
index 35de1554a529..6dbc70592219 100644
--- a/src/types/index.ts
+++ b/src/types/index.ts
@@ -825,7 +825,19 @@ export const TestSuiteConfigSchema = z.object({
   nunjucksFilters: z.record(z.string(), z.string()).optional(),
 
   // Envvar overrides
-  env: z.union([ProviderEnvOverridesSchema, z.record(z.string(), z.string())]).optional(),
+  env: z
+    .union([
+      ProviderEnvOverridesSchema,
+      z.record(
+        z.string(),
+        z.union([
+          z.string(),
+          z.number().transform((n) => String(n)),
+          z.boolean().transform((b) => String(b)),
+        ]),
+      ),
+    ])
+    .optional(),
 
   // Metrics to calculate after the eval has been completed
   derivedMetrics: z.array(DerivedMetricSchema).optional(),
diff --git a/test/types.test.ts b/test/types.test.ts
index bc04dfb87454..d9bc189e36db 100644
--- a/test/types.test.ts
+++ b/test/types.test.ts
@@ -177,6 +177,50 @@ describe('TestSuiteConfigSchema', () => {
     expect(configFiles.length).toBeGreaterThan(0);
   });
 
+  it('should validate env field with primitive values that get transformed to strings', () => {
+    const customEnvVars = {
+      CUSTOM_STRING_VALUE: 'string-value',
+      CUSTOM_NUMBER_VALUE: 42,
+      CUSTOM_BOOLEAN_TRUE: true,
+      CUSTOM_BOOLEAN_FALSE: false,
+    };
+
+    const recordSchema = z.record(
+      z.string(),
+      z.union([z.string(), z.number(), z.boolean()]).transform(String),
+    );
+    const recordResult = recordSchema.safeParse(customEnvVars);
+
+    expect(recordResult.success).toBe(true);
+
+    const parsedData = recordResult.success ? recordResult.data : {};
+
+    expect(parsedData.CUSTOM_STRING_VALUE).toBe('string-value');
+    expect(parsedData.CUSTOM_NUMBER_VALUE).toBe('42');
+    expect(parsedData.CUSTOM_BOOLEAN_TRUE).toBe('true');
+    expect(parsedData.CUSTOM_BOOLEAN_FALSE).toBe('false');
+
+    const config = {
+      providers: [{ id: 'test-provider' }],
+      prompts: ['test prompt'],
+      env: customEnvVars,
+    };
+
+    const result = TestSuiteConfigSchema.safeParse(config);
+
+    expect(result.success).toBe(true);
+
+    const testProvider = result.success
+      ? {
+          id: 'test-provider',
+          config: { someConfig: true },
+          env: result.data.env,
+        }
+      : { id: 'test-provider', config: { someConfig: true } };
+
+    expect(Object.keys(testProvider)).toContain('env');
+  });
+
   for (const file of configFiles) {
     it(`should validate ${path.relative(rootDir, file)}`, async () => {
       const configContent = fs.readFileSync(file, 'utf8');
diff --git a/test/types/index.test.ts b/test/types/index.test.ts
index d0426eb99b20..299726848d0d 100644
--- a/test/types/index.test.ts
+++ b/test/types/index.test.ts
@@ -1,4 +1,9 @@
-import { isGradingResult, TestCaseSchema, CommandLineOptionsSchema } from '../../src/types';
+import {
+  isGradingResult,
+  TestCaseSchema,
+  CommandLineOptionsSchema,
+  TestSuiteConfigSchema,
+} from '../../src/types';
 
 describe('isGradingResult', () => {
   it('should return true for valid grading result object', () => {
@@ -243,3 +248,49 @@ describe('CommandLineOptionsSchema', () => {
     );
   });
 });
+
+describe('TestSuiteConfigSchema env property', () => {
+  it('should validate config with string env values', () => {
+    const config = {
+      providers: ['provider1'],
+      prompts: ['prompt1'],
+      env: {
+        API_KEY: 'abc123',
+        DEBUG: 'true',
+      },
+    };
+    expect(() => TestSuiteConfigSchema.parse(config)).not.toThrow();
+  });
+
+  it('should validate config with number env values converted to strings', () => {
+    const config = {
+      providers: ['provider1'],
+      prompts: ['prompt1'],
+      env: {
+        PORT: 3000,
+        TIMEOUT: 5000,
+      },
+    };
+    expect(() => TestSuiteConfigSchema.parse(config)).not.toThrow();
+  });
+
+  it('should validate config with boolean env values converted to strings', () => {
+    const config = {
+      providers: ['provider1'],
+      prompts: ['prompt1'],
+      env: {
+        DEBUG: true,
+        VERBOSE: false,
+      },
+    };
+    expect(() => TestSuiteConfigSchema.parse(config)).not.toThrow();
+  });
+
+  it('should validate config with undefined env property', () => {
+    const config = {
+      providers: ['provider1'],
+      prompts: ['prompt1'],
+    };
+    expect(() => TestSuiteConfigSchema.parse(config)).not.toThrow();
+  });
+});


Reviews:
user: login: ellipsis-dev[bot]
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T00:53:29Z

Review comments:
diff_hunk: @@ -868,9 +868,7 @@
             },
             {
               "type": "object",
-              "additionalProperties": {
-                "type": "string"
-              }
+              "additionalProperties": {}
path: site/static/config-schema.json
user: login: ellipsis-dev[bot]
body: Relaxing `additionalProperties` to `{}` lets non-string values pass, since transformation is handled later. Ensure consumers expect transformed strings.
created_at: 2025-03-04T00:53:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -825,7 +825,9 @@ export const TestSuiteConfigSchema = z.object({
   nunjucksFilters: z.record(z.string(), z.string()).optional(),
 
   // Envvar overrides
-  env: z.union([ProviderEnvOverridesSchema, z.record(z.string(), z.string())]).optional(),
+  env: z
+    .union([ProviderEnvOverridesSchema, z.record(z.string(), z.any().transform(String))])
path: src/types/index.ts
user: login: ellipsis-dev[bot]
body: Consider checking for null/undefined when using `z.any().transform(String)` to convert env var values.
```suggestion
    .union([ProviderEnvOverridesSchema, z.record(z.string(), z.any().transform(value => value == null ? '' : String(value)))])
```
created_at: 2025-03-04T00:53:29Z
author_association: CONTRIBUTOR

PR comments:
user: login: gru-agent[bot]
created_at: 2025-03-04T00:47:01Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/20c8fc5a-c4b0-4a6e-8d1a-6bdadeb4fcd2) | 80ed28a5e0310e102b7d5549658105cf38974bfd | 🚫 Skipped | No files need to be tested {"site/static/config-schema.json":"target file(site/static/config-schema.json) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","src/types/index.ts":"target file(src/types/index.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","test/types.test.ts":"target file(test/types.test.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js"} |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR
user: login: gru-agent[bot]
created_at: 2025-03-04T00:47:04Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/6b9f3eca-4fe9-40fb-86d9-60e09c26ae08) | 80ed28a5e0310e102b7d5549658105cf38974bfd | ✅ Finished |  |

### Files

| File | Pull Request |
| ---- | ------------ |
| src/types/index.ts | 🟣 Merged https://github.com/promptfoo/promptfoo/pull/3274 |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR