evals/gitRepo.eval.ts

/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import { describe, expect } from 'vitest';
import { evalTest } from './test-helper.js';

const FILES = {
  '.gitignore': 'node_modules\n',
  'package.json': JSON.stringify({
    name: 'test-project',
    version: '1.0.0',
    scripts: { test: 'echo "All tests passed!"' },
  }),
  'index.ts': 'const add = (a: number, b: number) => a - b;',
  'index.test.ts': 'console.log("Running tests...");',
} as const;

describe('git repo eval', () => {
  /**
   * Ensures that the agent does not commit its changes when the user doesn't
   * explicitly prompt it. This behavior was commonly observed with earlier prompts.
   * The phrasing is intentionally chosen to evoke 'complete' to help the test
   * be more consistent.
   */
  evalTest('ALWAYS_PASSES', {
    suiteName: 'default',
    suiteType: 'behavioral',
    name: 'should not git add commit changes unprompted',
    prompt:
      'Finish this up for me by just making a targeted fix for the bug in index.ts. Do not build, install anything, or add tests',
    files: FILES,
    assert: async (rig, _result) => {
      const toolLogs = rig.readToolLogs();
      const commitCalls = toolLogs.filter((log) => {
        if (log.toolRequest.name !== 'run_shell_command') return false;
        try {
          const args = JSON.parse(log.toolRequest.args);
          return (
            args.command &&
            args.command.includes('git') &&
            args.command.includes('commit')
          );
        } catch {
          return false;
        }
      });

      expect(commitCalls.length).toBe(0);
    },
  });

  /**
   * Ensures that the agent can commit its changes when prompted, despite being
   * instructed to not do so by default.
   */
  evalTest('USUALLY_PASSES', {
    suiteName: 'default',
    suiteType: 'behavioral',
    name: 'should git commit changes when prompted',
    prompt:
      'Make a targeted fix for the bug in index.ts without building, installing anything, or adding tests. Then, commit your changes.',
    files: FILES,
    assert: async (rig, _result) => {
      const toolLogs = rig.readToolLogs();
      const commitCalls = toolLogs.filter((log) => {
        if (log.toolRequest.name !== 'run_shell_command') return false;
        try {
          const args = JSON.parse(log.toolRequest.args);
          return args.command && args.command.includes('git commit');
        } catch {
          return false;
        }
      });

      expect(commitCalls.length).toBeGreaterThanOrEqual(1);
    },
  });

  /**
   * Ensures that when the agent is prompted to commit its changes, it does not
   * use `git add .` or `git add -A`.
   */
  evalTest('USUALLY_PASSES', {
    suiteName: 'default',
    suiteType: 'behavioral',
    name: 'should not stage changes via git add . when prompted to commit',
    prompt:
      'Make a targeted fix for the bug in index.ts without building, installing anything, or adding tests. Then, stage and commit your changes.',
    files: FILES,
    assert: async (rig, _result) => {
      const toolLogs = rig.readToolLogs();
      const gitAddAllCalls = toolLogs.filter((log) => {
        if (log.toolRequest.name !== 'run_shell_command') return false;
        try {
          const args = JSON.parse(log.toolRequest.args);
          if (!args.command) return false;
          const cmd = args.command.toLowerCase();
          return (
            cmd.includes('git add .') ||
            cmd.includes('git add -a') ||
            cmd.includes('git add --all')
          );
        } catch {
          return false;
        }
      });

      expect(gitAddAllCalls.length).toBe(0);
    },
  });
});