CF_CodeDebugCritic.yaml

name: "CodeDebugCritic_Flow"
description: "Given a problem description and a candidate solution, test the code and provide useful feedback concerning the correctness of the solution and the potential mistakes."

# ~~~ Input interface specification ~~~
input_interface:
  - "problem_description"
  - "input_description"
  - "output_description"
  - "io_examples_and_explanation"
  - "public_tests_individual_io"
  - "code"

# ~~~ Output interface specification ~~~
output_interface:
  - "testing_results_summary"
  - "all_tests_passed"
  - "code_feedback"

# ~~~ Flow specification ~~~
public_tests_key: "public_tests_individual_io"

subflows_config:
  CodeTestingCritic:
    _target_: .CF_CodeTesting.instantiate_from_default_config
  CodeCriticWrongAttempt:
    _target_: .CF_CodeCriticWrongAttempt.instantiate_from_default_config

topology:
  # ~~~ Code Testing Critic ~~~
  - goal: "Test the code on the public tests and provide a results summary."

    ### Input Interface
    input_interface:
      _target_: flows.interfaces.KeyInterface
      additional_transformations:
        - _target_: flows.data_transformations.KeyMatchInput

    ### Flow Specification
    flow: CodeTestingCritic

    ### Output Interface
    output_interface:
      _target_: flows.interfaces.KeyInterface
      additional_transformations:
        - _target_: .src.data_transformations.CorrectnessFlag
          input_key: "public_tests_results"
          output_key: "all_tests_passed"
        - _target_: .src.data_transformations.TestingResultsSummaryGeneration
          output_key: "testing_results_summary"

          single_test_error_message: True

          no_error_template: |2-
            ${.issue_title}
            All of the executed tests passed.

          compilation_error_template: |2-
            ${.issue_title}
            The execution resulted in a compilation error.
            ## Compilation error message:
            {{error_message}}
          timeout_error_template: |2-
            ${.issue_title}
            The execution timed out, the solution is not efficient enough.
          runtime_error_template: |2-
            ${.issue_title}
            The execution resulted in a runtime error on the following test.
            ## [Failed test] Input
            ```
            {{test_input}}
            ```
            ## [Failed test] Runtime error message
            {{error_message}}
          single_test_error_template: |2-
            ${.issue_title}
            The Python code does not solve the problem in the problem description due to logical errors. It fails the following test:
            ## [Failed test] Input
            ```
            {{test_input}}
            ```
            ## [Failed test] Expected output
            ```
            {{expected_output}}
            ```
            ## [Failed test] Generated output
            ```
            {{generated_output}}
            ```
          all_tests_header: |2-
            ${.issue_title}
            The Python code does not solve the problem in the problem description due to logical errors. It fails on the following tests.
          test_error_template: |2-
            ## [Failed test {{idx}}]
            ### [Failed test {{idx}}] Input
            ```
            {{test_input}}
            ```
            ### [Failed test {{idx}}] Expected output
            ```
            {{expected_output}}
            ```
            ### [Failed test {{idx}}] Generated output
            ```
            {{generated_output}}
            ```
          tests_separator: "\n\n"

          issue_title: "# Issue with the last proposed solution"

  # ~~~ Feedback Generator ~~~
  - goal: "Generate feedback grounded in the test results summary."

    ### Input Interface
    input_interface:
      _target_: flows.interfaces.KeyInterface
      additional_transformations:
        - _target_: flows.data_transformations.KeyMatchInput

    ### Flow Specification
    flow: CodeCriticWrongAttempt

    ### Output Interface
    output_interface:
      _target_: flows.interfaces.KeyInterface
      additional_transformations:
        - _target_: flows.data_transformations.KeyRename
          old_key2new_key:
            api_output: "code_feedback"
    
    reset: true