Repository: UKGovernmentBEIS/inspect_ai
Issue: How to upload multiple audio files?
Root cause: Documentation Deficiency

I am currently processing the files as below:

```
def get_file_paths(directory):
    file_paths = []
    for root, _, files in os.walk(directory):
        for file in files:
            if file.endswith(".wav"):
                # Use os.path.join(root, file) to get the full path
                full_path = os.path.join(root, file)
                # Use os.path.relpath to get the path relative to the current directory
                relative_path = os.path.relpath(full_path, os.getcwd())
                file_paths.append(relative_path)
    return file_paths

def generate_pairwise_pairs(root_directory):
    all_paths = []
    for subdir in os.listdir(root_directory):
        subdir_path = os.path.join(root_directory, subdir)
        if os.path.isdir(subdir_path):
            file_paths = get_file_paths(subdir_path)
            all_paths.extend(file_paths[:1])  # Picks the first 1 files from each speaker

    samples = []

    for i in tqdm(range(len(all_paths))):
        for j in range(i + 1, len(all_paths)):
            if os.path.dirname(all_paths[i]) == os.path.dirname(all_paths[j]):
                samples.append(
                    Sample(
                        input="Compare the volumes of the two audio files",
                        choices=["yes", "no"],
                        target="yes",
                        files={"audio1": all_paths[i], "audio2": all_paths[j]},
                    )
                )
            else:
                samples.append(
                    Sample(
                        input="Compare the volumes of the two audio files",
                        choices=["yes", "no"],
                        target="no",
                        files={"audio1": all_paths[i], "audio2": all_paths[j]},
                    )
                )

    return MemoryDataset(samples=samples, name="dataset")
```

Then the call looks like this:

```
@task
def audios():
    return Task(
        dataset=generate_pairwise_pairs("audio_files"),
        plan=[system_message(SYSTEM_MESSAGE), generate()],
        scorer=match(),
    )
```

I inspected the dataset sample and it looks like this:

```
input='Compare the volumes of the two audio files' choices=['one', 'two'] target='one' id=None metadata=None files={'audio1': 'path/078.wav', 'audio2': 'path/1078.wav'} setup=None
```

A head of the log file looks like this when I use the gemini flash 1.5 model:

```
{
  "version": 2,
  "status": "success",
  "eval": {
    "task": "audios",
    "task_version": 0,
    "task_file": "task_agent.py",
    "task_id": "fATtdyE2iAD8xhrGnW78VH",
    "run_id": "mJJUoPhtfQw8veFFz7vf2a",
    "created": "2024-08-29T19:51:26-04:00",
    "dataset": {
      "name": "dataset",
      "samples": 10,
      "shuffled": false
    },
    "model": "google/gemini-1.5-flash",
    "task_attribs": {},
    "task_args": {},
    "model_args": {},
    "config": {},
    "packages": {
      "inspect_ai": "0.3.25"
    }
  },
  "plan": {
    "name": "plan",
    "steps": [
      {
        "solver": "system_message",
        "params": {
          "message": "For the following exercise only answer with a \"one\" or \"two\"."
        }
      },
      {
        "solver": "generate",
        "params": {}
      }
    ],
    "config": {}
  },
  "results": {
    "total_samples": 10,
    "completed_samples": 10,
    "scores": [
      {
        "name": "match",
        "scorer": "match",
        "params": {},
        "metrics": {
          "accuracy": {
            "name": "accuracy",
            "value": 0.0,
            "options": {}
          },
          "stderr": {
            "name": "stderr",
            "value": 0.0,
            "options": {}
          }
        }
      }
    ]
  },
  "stats": {
    "started_at": "2024-08-29T19:51:26-04:00",
    "completed_at": "2024-08-29T19:51:27-04:00",
    "model_usage": {
      "google/gemini-1.5-flash": {
        "input_tokens": 260,
        "output_tokens": 160,
        "total_tokens": 420
      }
    }
  },
  "samples": [
    {
      "id": 1,
      "epoch": 1,
      "input": "Compare the volumes of the two audio files",
      "choices": [
        "one",
        "two"
      ],
      "target": "one",
      "messages": [
        {
          "content": "For the following exercise only answer with a \"one\" or \"two\".",
          "role": "system"
        },
        {
          "content": "Compare the volume of the two audio files",
          "source": "input",
          "role": "user"
        },
        {
          "content": "Please provide the audio clips. I need to hear them to answer your question. \n",
          "source": "generate",
          "role": "assistant"
        }
      ],
```

Can anyone help me here? It seems like the audio files are passed properly but they are not loading correctly

Those `files` in a sample are actually for copying to a sandbox environment (e.g. a docker container). That is described here: https://inspect.ai-safety-institute.org.uk/datasets.html#dataset-samples

For including content in the input itself, currently only text and images are supported (and that for only some models). Here are docs on that: https://inspect.ai-safety-institute.org.uk/datasets.html#image-input

From looking at the source code for the Google provider, it appears to me as if you could include an audio file as if it was an "image" and we would upload it to Google with the right mime type (whether it will actually process the audio file I am not sure!)

Parsing the files as images worked for me. Thanks you for the quick response.