setup_actions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55b5db25",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/boom/.pyenv/versions/rag/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "token = os.getenv(\"HUGGINGFACE_TOKEN\")\n",
    "\n",
    "from huggingface_hub import login\n",
    "\n",
    "login(token=os.environ[\"HUGGINGFACE_TOKEN\"])\n",
    "\n",
    "# # Run isort on all folders\n",
    "# isort .\n",
    "# # Run black on all folders\n",
    "# black .\n",
    "# # Run flake8 on all folders\n",
    "# flake8 .\n",
    "\n",
    "# # Run pylint on all folders\n",
    "# pylint ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e2f982e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function load_dataset in module datasets.load:\n",
      "\n",
      "load_dataset(path: str, name: Optional[str] = None, data_dir: Optional[str] = None, data_files: Union[str, collections.abc.Sequence[str], collections.abc.Mapping[str, Union[str, collections.abc.Sequence[str]]], NoneType] = None, split: Union[str, datasets.splits.Split, NoneType] = None, cache_dir: Optional[str] = None, features: Optional[datasets.features.features.Features] = None, download_config: Optional[datasets.download.download_config.DownloadConfig] = None, download_mode: Union[datasets.download.download_manager.DownloadMode, str, NoneType] = None, verification_mode: Union[datasets.utils.info_utils.VerificationMode, str, NoneType] = None, keep_in_memory: Optional[bool] = None, save_infos: bool = False, revision: Union[str, datasets.utils.version.Version, NoneType] = None, token: Union[bool, str, NoneType] = None, streaming: bool = False, num_proc: Optional[int] = None, storage_options: Optional[dict] = None, trust_remote_code: Optional[bool] = None, **config_kwargs) -> Union[datasets.dataset_dict.DatasetDict, datasets.arrow_dataset.Dataset, datasets.dataset_dict.IterableDatasetDict, datasets.iterable_dataset.IterableDataset]\n",
      "    Load a dataset from the Hugging Face Hub, or a local dataset.\n",
      "    \n",
      "    You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`].\n",
      "    \n",
      "    A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly\n",
      "    in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.)\n",
      "    \n",
      "    This function does the following under the hood:\n",
      "    \n",
      "        1. Load a dataset builder:\n",
      "    \n",
      "            * Find the most common data format in the dataset and pick its associated builder (JSON, CSV, Parquet, Webdataset, ImageFolder, AudioFolder, etc.)\n",
      "            * Find which file goes into which split (e.g. train/test) based on file and directory names or on the YAML configuration\n",
      "            * It is also possible to specify `data_files` manually, and which dataset builder to use (e.g. \"parquet\").\n",
      "    \n",
      "        2. Run the dataset builder:\n",
      "    \n",
      "            In the general case:\n",
      "    \n",
      "            * Download the data files from the dataset if they are not already available locally or cached.\n",
      "            * Process and cache the dataset in typed Arrow tables for caching.\n",
      "    \n",
      "                Arrow table are arbitrarily long, typed tables which can store nested objects and be mapped to numpy/pandas/python generic types.\n",
      "                They can be directly accessed from disk, loaded in RAM or even streamed over the web.\n",
      "    \n",
      "            In the streaming case:\n",
      "    \n",
      "            * Don't download or cache anything. Instead, the dataset is lazily loaded and will be streamed on-the-fly when iterating on it.\n",
      "    \n",
      "        3. Return a dataset built from the requested splits in `split` (default: all).\n",
      "    \n",
      "    It can also use a custom dataset builder if the dataset contains a dataset script, but this feature is mostly for backward compatibility.\n",
      "    In this case the dataset script file must be named after the dataset repository or directory and end with \".py\".\n",
      "    \n",
      "    Args:\n",
      "    \n",
      "        path (`str`):\n",
      "            Path or name of the dataset.\n",
      "    \n",
      "            - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`])\n",
      "              -> load the dataset from supported files in the repository (csv, json, parquet, etc.)\n",
      "              e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files.\n",
      "    \n",
      "            - if `path` is a local directory\n",
      "              -> load the dataset from supported files in the directory (csv, json, parquet, etc.)\n",
      "              e.g. `'./path/to/directory/with/my/csv/data'`.\n",
      "    \n",
      "            - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified\n",
      "              (available builders are \"json\", \"csv\", \"parquet\", \"arrow\", \"text\", \"xml\", \"webdataset\", \"imagefolder\", \"audiofolder\", \"videofolder\")\n",
      "              -> load the dataset from the files in `data_files` or `data_dir`\n",
      "              e.g. `'parquet'`.\n",
      "    \n",
      "            It can also point to a local dataset script but this is not recommended.\n",
      "        name (`str`, *optional*):\n",
      "            Defining the name of the dataset configuration.\n",
      "        data_dir (`str`, *optional*):\n",
      "            Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`,\n",
      "            the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory.\n",
      "        data_files (`str` or `Sequence` or `Mapping`, *optional*):\n",
      "            Path(s) to source data file(s).\n",
      "        split (`Split` or `str`):\n",
      "            Which split of the data to load.\n",
      "            If `None`, will return a `dict` with all splits (typically `datasets.Split.TRAIN` and `datasets.Split.TEST`).\n",
      "            If given, will return a single Dataset.\n",
      "            Splits can be combined and specified like in tensorflow-datasets.\n",
      "        cache_dir (`str`, *optional*):\n",
      "            Directory to read/write data. Defaults to `\"~/.cache/huggingface/datasets\"`.\n",
      "        features (`Features`, *optional*):\n",
      "            Set the features type to use for this dataset.\n",
      "        download_config ([`DownloadConfig`], *optional*):\n",
      "            Specific download configuration parameters.\n",
      "        download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):\n",
      "            Download/generate mode.\n",
      "        verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):\n",
      "            Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...).\n",
      "    \n",
      "            <Added version=\"2.9.1\"/>\n",
      "        keep_in_memory (`bool`, defaults to `None`):\n",
      "            Whether to copy the dataset in-memory. If `None`, the dataset\n",
      "            will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to\n",
      "            nonzero. See more details in the [improve performance](../cache#improve-performance) section.\n",
      "        save_infos (`bool`, defaults to `False`):\n",
      "            Save the dataset information (checksums/size/splits/...).\n",
      "        revision ([`Version`] or `str`, *optional*):\n",
      "            Version of the dataset script to load.\n",
      "            As datasets have their own git repository on the Datasets Hub, the default version \"main\" corresponds to their \"main\" branch.\n",
      "            You can specify a different version than the default \"main\" by using a commit SHA or a git tag of the dataset repository.\n",
      "        token (`str` or `bool`, *optional*):\n",
      "            Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.\n",
      "            If `True`, or not specified, will get token from `\"~/.huggingface\"`.\n",
      "        streaming (`bool`, defaults to `False`):\n",
      "            If set to `True`, don't download the data files. Instead, it streams the data progressively while\n",
      "            iterating on the dataset. An [`IterableDataset`] or [`IterableDatasetDict`] is returned instead in this case.\n",
      "    \n",
      "            Note that streaming works for datasets that use data formats that support being iterated over like txt, csv, jsonl for example.\n",
      "            Json files may be downloaded completely. Also streaming from remote zip or gzip files is supported but other compressed formats\n",
      "            like rar and xz are not yet supported. The tgz format doesn't allow streaming.\n",
      "        num_proc (`int`, *optional*, defaults to `None`):\n",
      "            Number of processes when downloading and generating the dataset locally.\n",
      "            Multiprocessing is disabled by default.\n",
      "    \n",
      "            <Added version=\"2.7.0\"/>\n",
      "        storage_options (`dict`, *optional*, defaults to `None`):\n",
      "            **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any.\n",
      "    \n",
      "            <Added version=\"2.11.0\"/>\n",
      "        trust_remote_code (`bool`, *optional*, defaults to `None`):\n",
      "            Whether or not to allow for datasets defined on the Hub using a dataset script. This option\n",
      "            should only be set to `True` for repositories you trust and in which you have read the code, as it will\n",
      "            execute code present on the Hub on your local machine.\n",
      "    \n",
      "            <Added version=\"2.16.0\"/>\n",
      "    \n",
      "            <Changed version=\"2.20.0\">\n",
      "    \n",
      "            `trust_remote_code` defaults to `False` if not specified.\n",
      "    \n",
      "            </Changed>\n",
      "    \n",
      "        **config_kwargs (additional keyword arguments):\n",
      "            Keyword arguments to be passed to the `BuilderConfig`\n",
      "            and used in the [`DatasetBuilder`].\n",
      "    \n",
      "    Returns:\n",
      "        [`Dataset`] or [`DatasetDict`]:\n",
      "        - if `split` is not `None`: the dataset requested,\n",
      "        - if `split` is `None`, a [`~datasets.DatasetDict`] with each split.\n",
      "    \n",
      "        or [`IterableDataset`] or [`IterableDatasetDict`]: if `streaming=True`\n",
      "    \n",
      "        - if `split` is not `None`, the dataset is requested\n",
      "        - if `split` is `None`, a [`~datasets.streaming.IterableDatasetDict`] with each split.\n",
      "    \n",
      "    Example:\n",
      "    \n",
      "    Load a dataset from the Hugging Face Hub:\n",
      "    \n",
      "    ```py\n",
      "    >>> from datasets import load_dataset\n",
      "    >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train')\n",
      "    \n",
      "    # Load a subset or dataset configuration (here 'sst2')\n",
      "    >>> from datasets import load_dataset\n",
      "    >>> ds = load_dataset('nyu-mll/glue', 'sst2', split='train')\n",
      "    \n",
      "    # Manual mapping of data files to splits\n",
      "    >>> data_files = {'train': 'train.csv', 'test': 'test.csv'}\n",
      "    >>> ds = load_dataset('namespace/your_dataset_name', data_files=data_files)\n",
      "    \n",
      "    # Manual selection of a directory to load\n",
      "    >>> ds = load_dataset('namespace/your_dataset_name', data_dir='folder_name')\n",
      "    ```\n",
      "    \n",
      "    Load a local dataset:\n",
      "    \n",
      "    ```py\n",
      "    # Load a CSV file\n",
      "    >>> from datasets import load_dataset\n",
      "    >>> ds = load_dataset('csv', data_files='path/to/local/my_dataset.csv')\n",
      "    \n",
      "    # Load a JSON file\n",
      "    >>> from datasets import load_dataset\n",
      "    >>> ds = load_dataset('json', data_files='path/to/local/my_dataset.json')\n",
      "    \n",
      "    # Load from a local loading script (not recommended)\n",
      "    >>> from datasets import load_dataset\n",
      "    >>> ds = load_dataset('path/to/local/loading_script/loading_script.py', split='train')\n",
      "    ```\n",
      "    \n",
      "    Load an [`~datasets.IterableDataset`]:\n",
      "    \n",
      "    ```py\n",
      "    >>> from datasets import load_dataset\n",
      "    >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train', streaming=True)\n",
      "    ```\n",
      "    \n",
      "    Load an image dataset with the `ImageFolder` dataset builder:\n",
      "    \n",
      "    ```py\n",
      "    >>> from datasets import load_dataset\n",
      "    >>> ds = load_dataset('imagefolder', data_dir='/path/to/images', split='train')\n",
      "    ```\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(load_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "79deee46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating test split: 93 examples [00:00, 10442.25 examples/s]\n",
      "Generating validation split: 53 examples [00:00, 7329.56 examples/s]\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\n",
    "    \"gaia-benchmark/GAIA\",\n",
    "    name=\"2023_level1\",\n",
    "    split=\"validation\",\n",
    "    trust_remote_code=True,\n",
    "    cache_dir=\"ragdata\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "752d0bfa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dataset_info.json', 'gaia-validation.arrow', 'gaia-test.arrow']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(\n",
    "    r\"ragdata/gaia-benchmark___gaia/2023_level1/0.0.1/ec492fe4320ee795b1aed6bb46229c5f693226b0f1316347501c24b4baeee005\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "521be8df",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_15347/1929323449.py:43: LangChainDeprecationWarning: Since Chroma 0.4.x the manual persistence method is no longer supported as docs are automatically persisted.\n",
      "  vectorstore.persist()\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "from langchain.schema import Document\n",
    "from langchain.vectorstores import Chroma\n",
    "\n",
    "# Load the GAIA validation dataset\n",
    "dataset = load_dataset(\n",
    "    \"gaia-benchmark/GAIA\",\n",
    "    name=\"2023_level1\",\n",
    "    split=\"validation\",\n",
    "    trust_remote_code=True,\n",
    "    cache_dir=\"ragdata\",\n",
    ")\n",
    "# Prepare the embeddings model\n",
    "embeddings = HuggingFaceEmbeddings(model_name=\"all-MiniLM-L6-v2\")\n",
    "\n",
    "# Extract questions and their answers\n",
    "documents = []\n",
    "for entry in dataset:\n",
    "    question = entry[\"Question\"]\n",
    "    answer = entry[\"Final answer\"]\n",
    "\n",
    "    # Create a document with both the question and the answer as metadata\n",
    "    metadata = {\n",
    "        \"task_id\": entry[\"task_id\"],\n",
    "        \"steps\": entry[\"Annotator Metadata\"][\"Steps\"],\n",
    "        \"tools\": entry[\"Annotator Metadata\"][\"Tools\"],\n",
    "        \"answer\": answer,\n",
    "    }\n",
    "\n",
    "    # Add the question to the list of documents\n",
    "    documents.append(Document(page_content=question, metadata=metadata))\n",
    "\n",
    "# Insert the documents into Chroma\n",
    "vectorstore = Chroma.from_documents(\n",
    "    documents=documents,\n",
    "    embedding=embeddings,\n",
    "    collection_name=\"gaia_validation\",\n",
    "    persist_directory=\"./chroma_store\",\n",
    ")\n",
    "\n",
    "# Persist the data for future use\n",
    "vectorstore.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "210ea883",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'tools': ['1. A calculator', '3. Video recognition tools', '1. A web browser.', '2. google search', '2. Web browser', '1. web browser', '2. calculator', '2. A speech-to-text tool', '3. Calculator', '1. Wikipedia', '3. Audio capability', '3. Color recognition', '1. Markdown', '2. Video processing software', '1. search engine', 'No tools required', '2. Video parsing', '1. A word reversal tool / script', '1. Image recognition tools', '2. Image recognition', '2. Calculator', '2. search engine', '2. Search engine', '1. Access to Excel files', '1. Web browser', '1. Calculator', '1. Search engine', '1. Text Editor', '1. PowerPoint viewer', '3. PDF access', '1. image recognition/OCR', '2. A search engine.', '3. Audio processing software', '1. Excel', '3. PDF viewer', '3. A calculator.', \"1. Rubik's cube model\", '1. Word document access', 'None', '1. A file interface', '3. Access to academic journal websites', '2. Color recognition', '3. Calculator (or ability to count)', '1. Python', '2. A speech-to-text audio processing tool']}\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "# Load the GAIA validation dataset\n",
    "dataset = load_dataset(\"gaia-benchmark/GAIA\", name=\"2023_level1\", split=\"validation\")\n",
    "\n",
    "# Initialize a set to store unique tools\n",
    "unique_tools = set()\n",
    "\n",
    "# Iterate over each entry in the dataset\n",
    "for entry in dataset:\n",
    "    # Access the tools used (they are stored in the 'Tools' field of 'Annotator Metadata')\n",
    "    tools = entry[\"Annotator Metadata\"][\"Tools\"]\n",
    "\n",
    "    # Split the tools into a list (since they are stored as a string, we split by line breaks)\n",
    "    tools_list = tools.split(\"\\n\")\n",
    "\n",
    "    # Add each tool to the set (set automatically ensures uniqueness)\n",
    "    for tool in tools_list:\n",
    "        unique_tools.add(tool.strip())  # Remove any extra spaces or newlines\n",
    "\n",
    "# Convert the set of unique tools to a dictionary under the key 'tools'\n",
    "tools_dict = {\"tools\": list(unique_tools)}\n",
    "\n",
    "# Print the unique tools to get a sense of what was used\n",
    "print(tools_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55b688cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rag",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}