Repository: promptfoo/promptfoo
Pull request #66 (2023-07-13 17:38:33)

Add Azure OpenAI Provider
Hello! This project has been very useful for testing prompts in a structured way. Thanks for creating it!

There are some small features I'd like to add, first starting with Azure OpenAI.

- Added Azure OpenAI providers (i;e `azureopenai:chat:<name of deployment>`)
- Updated 'local' script (I added `--files` as it did not seem)


Patch:
diff --git a/examples/langchain-python/README.md b/examples/langchain-python/README.md
index 6921431f3..2499b6c51 100644
--- a/examples/langchain-python/README.md
+++ b/examples/langchain-python/README.md
@@ -1,8 +1,9 @@
-This example shows how to run a Python (LangChain) chain implementation with Promptfoo.  It compares GPT-4 versus LangChain's LLM Math module.
+This example shows how to run a Python (LangChain) chain implementation with Promptfoo. It compares GPT-4 versus LangChain's LLM Math module.
 
 To run it, first create a virtual env and install the requirements:
 
 Then activate the virtual env.
+
 ```
 python3 -m venv venv
 source venv/bin/activate
@@ -10,7 +11,7 @@ pip install -r requirements.txt
 ```
 
 Then run the eval:
+
 ```
 npx promptfoo eval
 ```
-
diff --git a/examples/openai-multiline-yaml/README.md b/examples/openai-multiline-yaml/README.md
new file mode 100644
index 000000000..0676e5bd9
--- /dev/null
+++ b/examples/openai-multiline-yaml/README.md
@@ -0,0 +1,5 @@
+This example is pre-configured in `promptfooconfig.yaml`. That means you can just run:
+
+```
+promptfoo eval
+```
diff --git a/examples/openai-multiline-yaml/prompt.yaml b/examples/openai-multiline-yaml/prompt.yaml
new file mode 100644
index 000000000..43de0f8d1
--- /dev/null
+++ b/examples/openai-multiline-yaml/prompt.yaml
@@ -0,0 +1,11 @@
+- role: system
+  content: |
+    Respond to the user while keeping the following in mind
+      - You are a travel agent
+      - You can help with logistics
+      - You do not refer to yourself as an AI unless asked
+
+    Be extremely conise.
+
+- role: user
+  content: {{content}}
diff --git a/examples/openai-multiline-yaml/promptfooconfig.yaml b/examples/openai-multiline-yaml/promptfooconfig.yaml
new file mode 100644
index 000000000..2093778f5
--- /dev/null
+++ b/examples/openai-multiline-yaml/promptfooconfig.yaml
@@ -0,0 +1,11 @@
+# prettier-ignore
+prompts: [prompt.yaml]
+providers: ['openai:chat:gpt-3.5-turbo-0613']
+
+tests:
+  - vars:
+      content: hey
+  - vars:
+      content: what can you help with?
+  - vars:
+      content: tell me about hawaii
diff --git a/package.json b/package.json
index 2c812b7af..ab4a4f8dc 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
   "name": "promptfoo",
   "description": "LLM eval & testing toolkit",
   "author": "Ian Webster",
-  "version": "0.17.4",
+  "version": "0.17.5",
   "license": "MIT",
   "type": "commonjs",
   "main": "dist/src/index.js",
@@ -25,7 +25,7 @@
     "promptfoo": "dist/src/main.js"
   },
   "scripts": {
-    "local": "ts-node --esm src/main.ts",
+    "local": "ts-node --esm --files src/main.ts",
     "install:client": "cd src/web/client && npm install",
     "build:clean": "rm -rf dist",
     "build:client": "cd src/web/client && npm run build && cp -r dist/ ../../../dist/src/web/client",
diff --git a/src/main.ts b/src/main.ts
index 33a0e81ef..b364f87c8 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -56,7 +56,11 @@ function createDummyFiles(directory: string | null) {
   writeFileSync(pathJoin(process.cwd(), directory, 'README.md'), DEFAULT_README);
 
   if (directory === '.') {
-    logger.info(chalk.green.bold('Wrote prompts.txt and promptfooconfig.yaml. Open README.md to get started!'));
+    logger.info(
+      chalk.green.bold(
+        'Wrote prompts.txt and promptfooconfig.yaml. Open README.md to get started!',
+      ),
+    );
   } else {
     logger.info(chalk.green.bold(`Wrote prompts.txt and promptfooconfig.yaml to ./${directory}`));
     logger.info(chalk.green(`\`cd ${directory}\` and open README.md to get started!`));
@@ -72,20 +76,20 @@ async function main() {
     pathJoin(pwd, 'promptfooconfig.json'),
     pathJoin(pwd, 'promptfooconfig.yaml'),
   ];
-  let config: Partial<UnifiedConfig> = {};
+  let defaultConfig: Partial<UnifiedConfig> = {};
   for (const path of potentialPaths) {
     const maybeConfig = await maybeReadConfig(path);
     if (maybeConfig) {
-      config = maybeConfig;
+      defaultConfig = maybeConfig;
       break;
     }
   }
 
   let evaluateOptions: EvaluateOptions = {};
-  if (config.evaluateOptions) {
-    evaluateOptions.generateSuggestions = config.evaluateOptions.generateSuggestions;
-    evaluateOptions.maxConcurrency = config.evaluateOptions.maxConcurrency;
-    evaluateOptions.showProgressBar = config.evaluateOptions.showProgressBar;
+  if (defaultConfig.evaluateOptions) {
+    evaluateOptions.generateSuggestions = defaultConfig.evaluateOptions.generateSuggestions;
+    evaluateOptions.maxConcurrency = defaultConfig.evaluateOptions.maxConcurrency;
+    evaluateOptions.showProgressBar = defaultConfig.evaluateOptions.showProgressBar;
   }
 
   const program = new Command();
@@ -159,28 +163,32 @@ async function main() {
   program
     .command('eval')
     .description('Evaluate prompts')
-    .option('-p, --prompts <paths...>', 'Paths to prompt files (.txt)', config.prompts)
+    .option('-p, --prompts <paths...>', 'Paths to prompt files (.txt)')
     .option(
       '-r, --providers <name or path...>',
       'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module',
     )
     .option(
       '-c, --config <path>',
-      'Path to configuration file. Automatically loads promptfooconfig.js/json/yaml',
+      'Path to configuration file. Automatically loads promptfoodefaultConfig.js/json/yaml',
     )
     .option(
       // TODO(ian): Remove `vars` for v1
       '-v, --vars, -t, --tests <path>',
       'Path to CSV with test cases',
-      config?.commandLineOptions?.vars,
+      defaultConfig?.commandLineOptions?.vars,
+    )
+    .option('-t, --tests <path>', 'Path to CSV with test cases')
+    .option(
+      '-o, --output <path>',
+      'Path to output file (csv, json, yaml, html)',
+      defaultConfig.outputPath,
     )
-    .option('-t, --tests <path>', 'Path to CSV with test cases', config?.commandLineOptions?.tests)
-    .option('-o, --output <path>', 'Path to output file (csv, json, yaml, html)', config.outputPath)
     .option(
       '-j, --max-concurrency <number>',
       'Maximum number of concurrent API calls',
-      config.evaluateOptions?.maxConcurrency
-        ? String(config.evaluateOptions.maxConcurrency)
+      defaultConfig.evaluateOptions?.maxConcurrency
+        ? String(defaultConfig.evaluateOptions.maxConcurrency)
         : undefined,
     )
     .option(
@@ -195,28 +203,28 @@ async function main() {
     .option(
       '--prompt-prefix <path>',
       'This prefix is prepended to every prompt',
-      config.defaultTest?.options?.prefix,
+      defaultConfig.defaultTest?.options?.prefix,
     )
     .option(
       '--prompt-suffix <path>',
       'This suffix is append to every prompt',
-      config.defaultTest?.options?.suffix,
+      defaultConfig.defaultTest?.options?.suffix,
     )
     .option(
       '--no-write',
       'Do not write results to promptfoo directory',
-      config?.commandLineOptions?.write,
+      defaultConfig?.commandLineOptions?.write,
     )
     .option(
       '--no-cache',
       'Do not read or write results to disk cache',
-      config?.commandLineOptions?.cache,
+      defaultConfig?.commandLineOptions?.cache,
     )
     .option('--no-progress-bar', 'Do not show progress bar')
-    .option('--no-table', 'Do not output table in CLI', config?.commandLineOptions?.table)
-    .option('--share', 'Create a shareable URL', config?.commandLineOptions?.share)
-    .option('--grader', 'Model that will grade outputs', config?.commandLineOptions?.grader)
-    .option('--verbose', 'Show debug logs', config?.commandLineOptions?.verbose)
+    .option('--no-table', 'Do not output table in CLI', defaultConfig?.commandLineOptions?.table)
+    .option('--share', 'Create a shareable URL', defaultConfig?.commandLineOptions?.share)
+    .option('--grader', 'Model that will grade outputs', defaultConfig?.commandLineOptions?.grader)
+    .option('--verbose', 'Show debug logs', defaultConfig?.commandLineOptions?.verbose)
     .option('--view [port]', 'View in browser ui')
     .action(async (cmdObj: CommandLineOptions & Command) => {
       // Misc settings
@@ -229,15 +237,16 @@ async function main() {
 
       // Config parsing
       const maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
+      let fileConfig: Partial<UnifiedConfig> = {};
       const configPath = cmdObj.config;
       if (configPath) {
-        config = await readConfig(configPath);
+        fileConfig = await readConfig(configPath);
       }
-      config = {
-        prompts: cmdObj.prompts || config.prompts,
-        providers: cmdObj.providers || config.providers,
-        tests: cmdObj.tests || cmdObj.vars || config.tests,
-        defaultTest: config.defaultTest,
+      const config: Partial<UnifiedConfig> = {
+        prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts,
+        providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers,
+        tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests,
+        defaultTest: fileConfig.defaultTest,
       };
 
       // Validation
diff --git a/src/providers.ts b/src/providers.ts
index 601284740..8c2834585 100644
--- a/src/providers.ts
+++ b/src/providers.ts
@@ -6,6 +6,7 @@ import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './provid
 import { AnthropicCompletionProvider } from './providers/anthropic';
 import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
 import { ScriptCompletionProvider } from './providers/scriptCompletion';
+import { AzureOpenAiChatCompletionProvider, AzureOpenAiCompletionProvider } from './providers/azureopenai'
 
 export async function loadApiProviders(
   providerPaths: ProviderId | ProviderId[] | RawProviderConfig[],
@@ -68,6 +69,29 @@ export async function loadApiProvider(
         `Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
       );
     }
+  } else if (providerPath?.startsWith('azureopenai:')) {
+    // Load Azure OpenAI module
+    const options = providerPath.split(':');
+    const modelType = options[1];
+    const deploymentName = options[2];
+
+    if (modelType === 'chat') {
+      return new AzureOpenAiChatCompletionProvider(
+        deploymentName,
+        undefined,
+        context?.config,
+      );
+    } else if (modelType === 'completion') {
+      return new AzureOpenAiCompletionProvider(
+        deploymentName,
+        undefined,
+        context?.config,
+      );
+    } else {
+      throw new Error(
+        `Unknown Azure OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
+      );
+    }
   } else if (providerPath?.startsWith('anthropic:')) {
     // Load Anthropic module
     const options = providerPath.split(':');
diff --git a/src/providers/anthropic.ts b/src/providers/anthropic.ts
index d37ea0664..83f7547fb 100644
--- a/src/providers/anthropic.ts
+++ b/src/providers/anthropic.ts
@@ -25,7 +25,7 @@ export class AnthropicCompletionProvider implements ApiProvider {
   constructor(modelName: string, apiKey?: string, context?: AnthropicCompletionOptions) {
     this.modelName = modelName;
     this.apiKey = apiKey || process.env.ANTHROPIC_API_KEY;
-    this.anthropic = new Anthropic({apiKey: this.apiKey});
+    this.anthropic = new Anthropic({ apiKey: this.apiKey });
     this.options = context || {};
   }
 
diff --git a/src/providers/azureopenai.ts b/src/providers/azureopenai.ts
new file mode 100644
index 000000000..a3c96cae4
--- /dev/null
+++ b/src/providers/azureopenai.ts
@@ -0,0 +1,275 @@
+import logger from '../logger';
+import { fetchJsonWithCache } from '../cache';
+import { REQUEST_TIMEOUT_MS } from './shared';
+
+import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
+
+interface AzureOpenAiCompletionOptions {
+  temperature?: number;
+  functions?: {
+    name: string;
+    description?: string;
+    parameters: any;
+  }[];
+  function_call?: 'none' | 'auto';
+}
+
+class AzureOpenAiGenericProvider implements ApiProvider {
+  deploymentName: string;
+  apiKey?: string;
+  apiHost: string;
+
+  constructor(deploymentName: string, apiKey?: string) {
+    this.deploymentName = deploymentName;
+
+    this.apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;
+
+    if (!process.env.AZURE_OPENAI_API_HOST) {
+      throw new Error('Azure OpenAI API host must be set');
+    }
+
+    this.apiHost = process.env.AZURE_OPENAI_API_HOST;
+  }
+
+  id(): string {
+    return `azureopenai:${this.deploymentName}`;
+  }
+
+  toString(): string {
+    return `[Azure OpenAI Provider ${this.deploymentName}]`;
+  }
+
+  // @ts-ignore: Prompt is not used in this implementation
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    throw new Error('Not implemented');
+  }
+}
+
+export class AzureOpenAiEmbeddingProvider extends AzureOpenAiGenericProvider {
+  async callEmbeddingApi(text: string): Promise<ProviderEmbeddingResponse> {
+    if (!this.apiKey) {
+      throw new Error('Azure OpenAI API key must be set for similarity comparison');
+    }
+
+    const body = {
+      input: text,
+      model: this.deploymentName,
+    };
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/embeddings?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            "api-key": this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+        tokenUsage: {
+          total: 0,
+          prompt: 0,
+          completion: 0,
+        },
+      };
+    }
+    logger.debug(`\tAzure OpenAI API response (embeddings): ${JSON.stringify(data)}`);
+
+    try {
+      const embedding = data?.data?.[0]?.embedding;
+      if (!embedding) {
+        throw new Error('No embedding returned');
+      }
+      const ret = {
+        embedding,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+      return ret;
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+        tokenUsage: {
+          total: data?.usage?.total_tokens,
+          prompt: data?.usage?.prompt_tokens,
+          completion: data?.usage?.completion_tokens,
+        },
+      };
+    }
+  }
+}
+
+export class AzureOpenAiCompletionProvider extends AzureOpenAiGenericProvider {
+  options: AzureOpenAiCompletionOptions;
+
+  constructor(deploymentName: string, apiKey?: string, context?: AzureOpenAiCompletionOptions) {
+    super(deploymentName, apiKey);
+    this.options = context || {};
+  }
+
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    if (!this.apiKey) {
+      throw new Error(
+        'Azure OpenAI API key is not set. Set AZURE_OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
+      );
+    }
+
+    let stop: string;
+    try {
+      stop = process.env.OPENAI_STOP
+        ? JSON.parse(process.env.OPENAI_STOP)
+        : ['
', '
'];
+    } catch (err) {
+      throw new Error(`OPENAI_STOP is not a valid JSON string: ${err}`);
+    }
+    const body = {
+      model: this.deploymentName,
+      prompt,
+      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      temperature:
+        options?.temperature ??
+        this.options.temperature ??
+        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      stop,
+    };
+    logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/completions?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            "api-key": this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+    logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
+    try {
+      return {
+        output: data.choices[0].text,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+      };
+    }
+  }
+}
+
+export class AzureOpenAiChatCompletionProvider extends AzureOpenAiGenericProvider {
+  options: AzureOpenAiCompletionOptions;
+
+  constructor(deploymentName: string, apiKey?: string, context?: AzureOpenAiCompletionOptions) {
+    super(deploymentName, apiKey);
+    this.options = context || {};
+  }
+
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    if (!this.apiKey) {
+      throw new Error(
+        'Azure OpenAI API key is not set. Set AZURE_OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
+      );
+    }
+
+    let messages: { role: string; content: string; name?: string }[];
+    try {
+      messages = JSON.parse(prompt) as { role: string; content: string }[];
+    } catch (err) {
+      const trimmedPrompt = prompt.trim();
+      if (
+        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
+        trimmedPrompt.startsWith('{') ||
+        trimmedPrompt.startsWith('[')
+      ) {
+        throw new Error(
+          `Azure OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+        );
+      }
+      messages = [{ role: 'user', content: prompt }];
+    }
+
+    const body = {
+      model: this.deploymentName,
+      messages: messages,
+      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      temperature:
+        options?.temperature ??
+        this.options.temperature ??
+        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      functions: options?.functions || this.options.functions || undefined,
+      function_call: options?.function_call || this.options.function_call || undefined,
+    };
+    logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
+
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/chat/completions?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            "api-key": this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+
+    logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
+    try {
+      const message = data.choices[0].message;
+      const output =
+        message.content === null ? JSON.stringify(message.function_call) : message.content;
+      return {
+        output,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+      };
+    }
+  }
+}
diff --git a/src/providers/openai.ts b/src/providers/openai.ts
index 8214fdc63..f21ce48fa 100644
--- a/src/providers/openai.ts
+++ b/src/providers/openai.ts
@@ -1,3 +1,5 @@
+import yaml from 'js-yaml';
+
 import logger from '../logger';
 import { fetchJsonWithCache } from '../cache';
 import { REQUEST_TIMEOUT_MS } from './shared';
@@ -226,20 +228,34 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     }
 
     let messages: { role: string; content: string; name?: string }[];
-    try {
-      messages = JSON.parse(prompt) as { role: string; content: string }[];
-    } catch (err) {
-      const trimmedPrompt = prompt.trim();
-      if (
-        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
-        trimmedPrompt.startsWith('{') ||
-        trimmedPrompt.startsWith('[')
-      ) {
+    const trimmedPrompt = prompt.trim();
+    if (trimmedPrompt.startsWith('- role:')) {
+      try {
+        // Try YAML
+        messages = yaml.load(prompt) as { role: string; content: string }[];
+      } catch (err) {
         throw new Error(
-          `OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+          `OpenAI Chat Completion prompt is not a valid YAML string: ${err}\n\n${prompt}`,
         );
       }
-      messages = [{ role: 'user', content: prompt }];
+    } else {
+      try {
+        // Try JSON
+        messages = JSON.parse(prompt) as { role: string; content: string }[];
+      } catch (err) {
+        if (
+          process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
+          trimmedPrompt.startsWith('{') ||
+          trimmedPrompt.startsWith('[')
+        ) {
+          throw new Error(
+            `OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+          );
+        }
+
+        // Fall back to wrapping the prompt in a user message
+        messages = [{ role: 'user', content: prompt }];
+      }
     }
 
     const body = {


Test patch:
diff --git a/test/providers.test.ts b/test/providers.test.ts
index 2cdd4a8b2..29f836bd1 100644
--- a/test/providers.test.ts
+++ b/test/providers.test.ts
@@ -6,6 +6,7 @@ import { AnthropicCompletionProvider } from '../src/providers/anthropic';
 import { disableCache, enableCache } from '../src/cache.js';
 import { loadApiProvider, loadApiProviders } from '../src/providers.js';
 import type { RawProviderConfig } from '../src/types';
+import { AzureOpenAiChatCompletionProvider, AzureOpenAiCompletionProvider } from '../src/providers/azureopenai';
 
 jest.mock('node-fetch', () => jest.fn());
 
@@ -75,11 +76,70 @@ describe('providers', () => {
     enableCache();
   });
 
+  test('AzureOpenAiCompletionProvider callApi', async () => {
+    const mockResponse = {
+      json: jest.fn().mockResolvedValue({
+        choices: [{ text: 'Test output' }],
+        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+      }),
+    };
+    (fetch as unknown as jest.Mock).mockResolvedValue(mockResponse);
+
+    const provider = new AzureOpenAiCompletionProvider('text-davinci-003', 'test-api-key');
+    const result = await provider.callApi('Test prompt');
+
+    expect(fetch).toHaveBeenCalledTimes(1);
+    expect(result.output).toBe('Test output');
+    expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+  });
+
+  test('AzureOpenAiChatCompletionProvider callApi', async () => {
+    const mockResponse = {
+      json: jest.fn().mockResolvedValue({
+        choices: [{ message: { content: 'Test output' } }],
+        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+      }),
+    };
+    (fetch as unknown as jest.Mock).mockResolvedValue(mockResponse);
+
+    const provider = new AzureOpenAiChatCompletionProvider('gpt-3.5-turbo', 'test-api-key');
+    const result = await provider.callApi(
+      JSON.stringify([{ role: 'user', content: 'Test prompt' }]),
+    );
+
+    expect(fetch).toHaveBeenCalledTimes(1);
+    expect(result.output).toBe('Test output');
+    expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+  });
+
+  test('AzureOpenAiChatCompletionProvider callApi with cache disabled', async () => {
+    disableCache();
+
+    const mockResponse = {
+      json: jest.fn().mockResolvedValue({
+        choices: [{ message: { content: 'Test output' } }],
+        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+      }),
+    };
+    (fetch as unknown as jest.Mock).mockResolvedValue(mockResponse);
+
+    const provider = new AzureOpenAiChatCompletionProvider('gpt-3.5-turbo', 'test-api-key');
+    const result = await provider.callApi(
+      JSON.stringify([{ role: 'user', content: 'Test prompt' }]),
+    );
+
+    expect(fetch).toHaveBeenCalledTimes(1);
+    expect(result.output).toBe('Test output');
+    expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+
+    enableCache();
+  });
+
   test('AnthropicCompletionProvider callApi', async () => {
     const provider = new AnthropicCompletionProvider('claude-1', 'test-api-key');
     provider.anthropic.completions.create = jest.fn().mockResolvedValue({
       completion: 'Test output',
-    })
+    });
     const result = await provider.callApi('Test prompt');
 
     expect(provider.anthropic.completions.create).toHaveBeenCalledTimes(1);
@@ -107,6 +167,17 @@ describe('providers', () => {
     expect(provider).toBeInstanceOf(OpenAiCompletionProvider);
   });
 
+  test('loadApiProvider with azureopenai:completion:modelName', async () => {
+    const provider = await loadApiProvider('azureopenai:completion:text-davinci-003');
+    expect(provider).toBeInstanceOf(AzureOpenAiCompletionProvider);
+  });
+
+  test('loadApiProvider with azureopenai:chat:modelName', async () => {
+    const provider = await loadApiProvider('azureopenai:chat:gpt-3.5-turbo');
+    expect(provider).toBeInstanceOf(AzureOpenAiChatCompletionProvider);
+  });
+
+
   test('loadApiProvider with anthropic:completion', async () => {
     const provider = await loadApiProvider('anthropic:completion');
     expect(provider).toBeInstanceOf(AnthropicCompletionProvider);