Repository: browserbase/stagehand
File: evals/evaluator.ts (TypeScript)

Code before:
  BatchEvaluateOptions,
  EvaluationResult,
} from "@/types/evaluator";
import { LLMParsedResponse } from "@/lib/inference";
import { LLMResponse } from "@/lib/llm/LLMClient";
import { LogLine } from "@/types/log";
import { z } from "zod";

dotenv.config();

const EvaluationSchema = z.object({
  evaluation: z.enum(["YES", "NO"]),
  reasoning: z.string(),
});

const BatchEvaluationSchema = z.array(EvaluationSchema);

export class Evaluator {
  private stagehand: Stagehand;
  private modelName: AvailableModel;
  private modelClientOptions: ClientOptions | { apiKey: string };
  private silentLogger: (message: LogLine) => void;

  constructor(
    stagehand: Stagehand,
    _modelName?: AvailableModel,
    modelClientOptions?: ClientOptions,
  ) {
    this.stagehand = stagehand;
    this.modelName = "google/gemini-2.5-flash";
    this.modelClientOptions = modelClientOptions || {
      apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY || "",
    };
  }

  async ask(options: EvaluateOptions): Promise<EvaluationResult> {
    const {
      question,
      answer,
      screenshot = true,
      systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and the state of a task (in the form of a screenshot, or an answer). Provide a detailed reasoning for your answer.
          Be critical about the question and the answer, the slightest detail might be the difference between yes and no.
          Today's date is ${new Date().toLocaleDateString()}`,
      screenshotDelayMs = 250,
    } = options;
    if (!question) {
      throw new Error("Question cannot be an empty string");
    }
    if (!answer && !screenshot) {
      throw new Error("Either answer (text) or screenshot must be provided");
    }

Review comment (suggestion):
```suggestion
    modelName?: AvailableModel,
```

Code after:
  BatchEvaluateOptions,
  EvaluationResult,
} from "@/types/evaluator";
import { LLMParsedResponse } from "@/lib/inference";
import { LLMResponse } from "@/lib/llm/LLMClient";
import { LogLine } from "@/types/log";
import { z } from "zod";

dotenv.config();

const EvaluationSchema = z.object({
  evaluation: z.enum(["YES", "NO"]),
  reasoning: z.string(),
});

const BatchEvaluationSchema = z.array(EvaluationSchema);

export class Evaluator {
  private stagehand: Stagehand;
  private modelName: AvailableModel;
  private modelClientOptions: ClientOptions | { apiKey: string };
  private silentLogger: (message: LogLine) => void;

  constructor(
    stagehand: Stagehand,
    modelName?: AvailableModel,
    modelClientOptions?: ClientOptions,
  ) {
    this.stagehand = stagehand;
    this.modelName = modelName || "google/gemini-2.5-flash";
    this.modelClientOptions = modelClientOptions || {
      apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY || "",
    };
  }

  async ask(options: EvaluateOptions): Promise<EvaluationResult> {
    const {
      question,
      answer,
      screenshot = true,
      systemPrompt = `You are an expert evaluator that confidently returns YES or NO given a question and the state of a task (in the form of a screenshot, or an answer). Provide a detailed reasoning for your answer.
          Be critical about the question and the answer, the slightest detail might be the difference between yes and no.
          Today's date is ${new Date().toLocaleDateString()}`,
      screenshotDelayMs = 250,
    } = options;
    if (!question) {
      throw new Error("Question cannot be an empty string");
    }
    if (!answer && !screenshot) {
      throw new Error("Either answer (text) or screenshot must be provided");
    }