Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1,4 @@
+.env
+.venv
+data
+__pycache__/

LICENCE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 vladbogo, albanie, ioanacroi, abuonomo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,31 @@
 ---
 title: Filtir
-emoji: 📚
-colorFrom: pink
-colorTo: yellow
+app_file: app.py
 sdk: gradio
 sdk_version: 4.19.0
-app_file: app.py
-pinned: false
 ---
+# Filtir - fact checking pipeline
+
+This repo contains the Filtir pipeline for claim extraction and fact-checking.
+
+## Prerequisites
+
+### Create and prepare venv
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Setup keys
+In order to run the code you need to set up the following keys and add them to .env:
+
+- OPENAI_API_KEY - used to call the OpenAI API
+- COHERE_API_KEY and WEAVIATE_API_KEY - used for Wikipedia search
+- GOOGLE_CLOUD_API_KEY and GOOGLE_CUSTOM_SEARCH_ENGINE_ID - used for Google search
+
+## Run the pipeline
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+```bash
+python run_pipeline.py --file example.txt --model gpt-4-1106-preview
+```

app.py

@@ -0,0 +1,30 @@
+# gradio_app.py
+
+import gradio as gr
+from run_pipeline import get_fact_checked
+
+
+def fact_check_function(text, model):
+    # Assume the text is already read from the user input, so we don't need to open a file here
+    out = get_fact_checked(text, mode="slow", model=model)
+    return out["fact_checked_md"]
+
+
+def create_gradio_interface():
+    iface = gr.Interface(
+        allow_flagging=False,
+        fn=fact_check_function,
+        inputs=[
+            gr.Textbox(
+                lines=10, placeholder="Enter text to fact-check...", label="Input Text"
+            ),
+            gr.Dropdown(choices=["gpt-4-1106-preview"], label="Model"),
+        ],
+        outputs=gr.Markdown(label="Filtir Output"),
+    )
+    return iface
+
+
+if __name__ == "__main__":
+    iface = create_gradio_interface()
+    iface.launch()

example.txt

@@ -0,0 +1 @@
+Arendt fled Germany in 1933.

fetch_evidence.py

@@ -0,0 +1,184 @@
+import requests
+from bs4 import BeautifulSoup
+import json
+import json5
+import argparse
+from pathlib import Path
+import multiprocessing as mp
+from zsvision.zs_multiproc import starmap_with_kwargs
+from pipeline_paths import PIPELINE_PATHS
+from datetime import datetime
+import urllib.robotparser
+import urllib.parse
+from utils import get_google_search_results
+
+import time
+from random import randint
+from fake_useragent import UserAgent
+from newspaper import Article, Config
+
+
+def can_scrape(url, user_agent="*"):
+    rp = urllib.robotparser.RobotFileParser()
+    rp.set_url(f"{url.scheme}://{url.netloc}/robots.txt")
+    # be conservative - if we can't find robots.txt, don't scrapes
+    try:
+        rp.read()
+        ok_to_scrape = rp.can_fetch(user_agent, url.geturl())
+    except urllib.error.URLError:
+        ok_to_scrape = False
+    return ok_to_scrape
+
+
+def fetch_search_results_to_gather_evidence(
+    args,
+    idx: int,
+    total: int,
+    search_results_dest_path: Path,
+    queryset: dict,
+):
+    user_agent = UserAgent()
+    config = Config()
+    config.fetch_images = False
+    print(f"Query {idx}/{total}")
+
+    search_results_dest_path.parent.mkdir(exist_ok=True, parents=True)
+
+    # check if we already have search_results for this title
+    if search_results_dest_path.exists() and not args.refresh:
+        print(f"Found existing search results at {search_results_dest_path}, skipping")
+        return 0
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+        "DNT": "1",
+        "Connection": "keep-alive",
+        "Upgrade-Insecure-Requests": "1",
+    }
+
+    # we assume some sites won't permit scraping, so we'll skip these
+    num_results = args.num_search_results_to_keep + 5
+    results = {}
+
+    for item in queryset:
+        if item["search_query"] == "no suitable query":
+            item["search_results"] = []
+            continue
+
+        search_results = get_google_search_results(
+            query_str=item["search_query"], num_results=num_results
+        )
+
+        if search_results == [{"Result": "No good Google Search Result was found"}]:
+            item["search_results"] = []
+            continue
+
+        parsed_results = []
+        for search_result in search_results:
+            if not can_scrape(
+                urllib.parse.urlparse(search_result["link"]), user_agent="MyScraper"
+            ):
+                print(
+                    f"Skipping {search_result['link']} because it doesn't permit scraping"
+                )
+                continue
+            try:
+                config.browser_user_agent = user_agent.random
+                article = Article(search_result["link"], language="en", config=config)
+                article.download()
+                article.parse()
+                text = article.text
+            except Exception as e:
+                print(f"Error parsing article: {e}, trying with requests.get...")
+                try:
+                    response = requests.get(
+                        search_result["link"], timeout=15, headers=headers
+                    )
+                    html = response.text
+                    soup = BeautifulSoup(html, features="html.parser")
+                    text = soup.get_text()
+                except Exception as exception:
+                    print(f"Error parsing article: {exception}")
+                    raise exception
+
+            search_result["text"] = text
+            parsed_results.append(search_result)
+            if len(parsed_results) == args.num_search_results_to_keep:
+                break
+        item["search_results"] = parsed_results
+
+    # update the queryset with new information
+    date_str = datetime.now().strftime("%Y-%m-%d")
+    results = {"documents": queryset, "dates": {"search_results_fetched": date_str}}
+
+    print(
+        f"Writing web pages for search results for {len(queryset)} queries to {search_results_dest_path}"
+    )
+    with open(search_results_dest_path, "w") as f:
+        f.write(json.dumps(results, indent=4, sort_keys=True))
+
+
+def main():
+    args = parse_args()
+    search_query_paths = list(
+        PIPELINE_PATHS["search_queries_for_evidence"].glob("**/*.json")
+    )
+
+    if args.limit:
+        print(f"Limited to {args.limit} search querysets")
+        search_query_paths = search_query_paths[: args.limit]
+
+    kwarg_list = []
+    for idx, search_query_path in enumerate(search_query_paths):
+        rel_path = search_query_path.relative_to(
+            PIPELINE_PATHS["search_queries_for_evidence"]
+        )
+        dest_path = PIPELINE_PATHS["google_search_results_evidence"] / rel_path
+
+        if dest_path.exists() and not args.refresh:
+            print(f"For {search_query_path}, found results at {dest_path}, skipping")
+            continue
+
+        with open(search_query_path, "r") as f:
+            queryset = json.load(f)
+            kwarg_list.append(
+                {
+                    "idx": idx,
+                    "total": len(search_query_paths),
+                    "search_results_dest_path": dest_path,
+                    "args": args,
+                    "queryset": queryset,
+                }
+            )
+
+    # provide the total number of queries to each process
+    for kwargs in kwarg_list:
+        kwargs["total"] = len(kwarg_list)
+
+    # single process
+    if args.processes == 1:
+        cost = 0
+        for kwargs in kwarg_list:
+            fetch_search_results_to_gather_evidence(**kwargs)
+    else:  # multiprocess
+        func = fetch_search_results_to_gather_evidence
+        with mp.Pool(processes=args.processes) as pool:
+            starmap_with_kwargs(pool=pool, func=func, kwargs_iter=kwarg_list)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model", default="gpt-3.5-turbo", choices=["gpt-4", "gpt-3.5-turbo"]
+    )
+    parser.add_argument("--limit", default=0, type=int)
+    parser.add_argument("--refresh", action="store_true")
+    parser.add_argument("--num_search_results_to_keep", type=int, default=3)
+    parser.add_argument("--processes", type=int, default=1)
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main()

flagged/log.csv

@@ -0,0 +1,5 @@
+Input Text,Model,Fact Checked Output,flag,username,timestamp
+,gpt-4-1106-preview,,,,2024-02-16 13:59:41.379589
+The earth is flat,gpt-4-1106-preview,"The earth is flat[^The-earth-is-flat]
+
+[^The-earth-is-flat]: ❗ Claim: The earth is flat. 👉 Unsupported ""Flat-Earth ideas are based on basic scientific misunderstandings that can be easily refuted."",""For most people, even those who have no physics background, the evidence for a spherical Earth is obvious."" The claim that the Earth is flat is directly contradicted by the evidence, which states that Flat-Earth ideas are based on basic scientific misunderstandings and that the evidence for a spherical Earth is obvious., URLs: https://physicsworld.com/a/fighting-flat-earth-theory/, date accessed: 2024-02-16 ",,,2024-02-16 14:00:40.524144

generate_search_queries.py

@@ -0,0 +1,137 @@
+import argparse
+from pathlib import Path
+import numpy as np
+from pipeline_paths import PIPELINE_PATHS
+import json
+from zsvision.zs_utils import BlockTimer
+from typing import Dict, List
+from llm_api_utils import (
+    call_openai_with_exponetial_backoff,
+    estimate_cost_of_text_generation_api_call,
+    init_openai_with_api_key,
+)
+
+
+def generate_search_queries(args, src_path: Path, dest_path: Path):
+    """
+    Generate a search query that can be used to verify a claim.
+    """
+    init_openai_with_api_key(api_key_path=args.api_key_path)
+    with open(src_path, "r") as f:
+        claims_and_sources = json.load(f)
+
+    # exclude subjective claims
+    original_num_claims = len(claims_and_sources)
+    claims_and_sources = [
+        claim_and_source
+        for claim_and_source in claims_and_sources
+        if claim_and_source["label"] == "objective"
+    ]
+    num_claims = len(claims_and_sources)
+    print(
+        f"Filtered from {original_num_claims} claims to {num_claims} objective claims"
+    )
+
+    # we limit the number of claims per api call (otherwise GPT-4 can choke)
+    num_batches = int(np.ceil(num_claims / args.max_claims_per_api_call))
+    claims_and_sources_batches = [
+        batch.tolist() for batch in np.array_split(claims_and_sources, num_batches)
+    ]
+    queries = []
+
+    all_claims_str = "\n".join([claim["claim"] for claim in claims_and_sources])
+
+    for idx, claims_and_sources_batch in enumerate(claims_and_sources_batches):
+        print(
+            f"Processing batch {idx+1} of {len(claims_and_sources_batches)} (containing {len(claims_and_sources_batch)} claims)"
+        )
+
+        claim_str = "\n".join([claim["claim"] for claim in claims_and_sources_batch])
+        num_batch_claims = len(claims_and_sources_batch)
+
+        # we provide the full list of claims as context (to help resolve ambiguity), but only ask for queries for the current batch
+        prompt = f"""\
+You are working as part of a team and your individual task is to help check a subset of the following claims:\n
+{all_claims_str}
+
+Your individual task is as follows. \
+For each of the {num_batch_claims} claims made below, provide a suitable Google search query that would enable a human to verify the claim. \
+Note that Google can perform calculations and conversions, so you can use it to check numerical claims. \
+If you think no Google query will be useful, then write "no suitable query". \
+Each proposed Google search query should be on a separate line (do not prefix your queries with bullet points or numbers). \
+There should be {num_batch_claims} queries in total.\n \
+
+{claim_str}
+"""
+        persona = "You are a careful research assistant who helps with fact-checking and editing informative articles."
+        system_message = {"role": "system", "content": persona}
+        user_message = {"role": "user", "content": prompt}
+        messages = [system_message, user_message]
+
+        with BlockTimer(f"Using OpenAI API to extract claims with {args.model}"):
+            response = call_openai_with_exponetial_backoff(
+                model=args.model,
+                temperature=args.temperature,
+                messages=messages,
+            )
+
+        cost = estimate_cost_of_text_generation_api_call(
+            model=args.model, response=response, verbose=True
+        )
+
+        proposed_queries = response.choices[0].message.content
+        batch_queries = proposed_queries.split("\n")
+        assert (
+            len(batch_queries) == num_batch_claims
+        ), f"Expected {num_batch_claims} queries, but got {len(queries)}"
+        print(f"Generated {len(batch_queries)} queries (cost: {cost:.4f} USD)")
+        queries.extend(batch_queries)
+
+    querysets = []
+    for claim_and_source, query in zip(claims_and_sources, queries):
+        queryset = {**claim_and_source, "search_query": query}
+        querysets.append(queryset)
+
+    dest_path.parent.mkdir(exist_ok=True, parents=True)
+    with open(dest_path, "w") as f:
+        json.dump(querysets, f, indent=4, sort_keys=True)
+
+
+def main():
+    args = parse_args()
+
+    src_paths = list(
+        PIPELINE_PATHS["extracted_claims_with_classifications_dir"].glob("**/*.json")
+    )
+    print(
+        f"Found {len(src_paths)} claim files in {PIPELINE_PATHS['extracted_claims_with_classifications_dir']}"
+    )
+    dest_dir = PIPELINE_PATHS["search_queries_for_evidence"]
+
+    for src_path in src_paths:
+        dest_path = dest_dir / src_path.relative_to(
+            PIPELINE_PATHS["extracted_claims_with_classifications_dir"]
+        )
+        if not dest_path.exists() or args.refresh:
+            generate_search_queries(
+                args=args,
+                src_path=src_path,
+                dest_path=dest_path,
+            )
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--temperature", type=float, default=0)
+    parser.add_argument(
+        "--model", default="gpt-3.5-turbo", choices=["gpt-4", "gpt-3.5-turbo"]
+    )
+    parser.add_argument("--dest_dir", default="data/search_queries", type=Path)
+    parser.add_argument("--api_key_path", default="OPENAI_API_KEY.txt")
+    parser.add_argument("--max_claims_per_api_call", type=int, default=10)
+    parser.add_argument("--refresh", action="store_true")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main()

llm_api_utils.py

@@ -0,0 +1,60 @@
+import random
+import cohere
+import os
+import openai
+import time
+import backoff
+
+
+PRICE_PER_1K_TOKENS = {
+    "gpt-4": {"prompt": 0.03, "completion": 0.06},
+    "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03},
+    "gpt-3.5-turbo": {"prompt": 0.001, "completion": 0.002},
+    "ada": {"embed": 0.0004},
+    "text-embedding-ada-002": {"embed": 0.0001},
+}
+
+
+EMBEDDING_DIMENSIONS = {
+    "ada": 1536,
+    "text-embedding-ada-002": 1536,
+}
+
+
+def estimate_cost_of_text_generation_api_call(
+    model: str, response: dict, verbose: bool
+) -> float:
+    completion_tokens = response.usage.completion_tokens
+    prompt_tokens = response.usage.prompt_tokens
+    total_tokens = response.usage.total_tokens
+
+    prompt_cost = prompt_tokens / 1000 * PRICE_PER_1K_TOKENS[model]["prompt"]
+    completion_cost = (
+        completion_tokens / 1000 * PRICE_PER_1K_TOKENS[model]["completion"]
+    )
+    cost = prompt_cost + completion_cost
+
+    if verbose:
+        summary = f"""\
+Used {prompt_tokens} prompt tokens, {completion_tokens} completion tokens, {total_tokens} total tokens
+Esimated cost: {cost:.4f} USD
+"""
+        print(summary)
+    return cost
+
+
+@backoff.on_exception(backoff.expo, (openai.RateLimitError, openai.APIConnectionError))
+def call_openai_with_exponetial_backoff(**kwargs):
+    rand_sleep_in_secs = 5 * random.random()
+    time.sleep(rand_sleep_in_secs)
+    return openai.chat.completions.create(**kwargs)
+
+
+def init_openai_with_api_key():
+    openai.api_key = os.environ.get("OPENAI_API_KEY")
+
+
+def init_cohere_with_api_key():
+    COHERE_API_KEY = os.environ.get("COHERE_API_KEY")
+    co = cohere.Client(COHERE_API_KEY)
+    return co

objective_claims.txt

@@ -0,0 +1,3 @@
+Hannah Arendt was born in 1906.
+Hannah Arendt was raised in a Jewish family.
+Arendt fled Germany in 1933.

pipeline_paths.py

@@ -0,0 +1,25 @@
+"""A module that contains the paths to the various files and folders used in the pipeline."""
+from pathlib import Path
+
+PIPELINE_PATHS = {
+    # google queries
+    "google_custom_search_engine_id_path": "google_custom_search_engine_id.txt",
+    # raw inputs to pipeline
+    "source_document_dir": "data/source_documents",
+    # claim extraction
+    "extracted_claims_dir": "data/extracted_claims",
+    "extracted_claims_with_anchor_fixes_dir": "data/extracted_claims_with_anchor_fixes",
+    "extracted_claims_with_classifications_dir": "data/extracted_with_classifications_claims",
+    "objective_claims_dir": "data/objective_claims",
+    # evidence gathering
+    "cohere_wikipedia_evidence": "data/evidence_gathering/cohere_wikipedia",
+    "google_search_results_evidence": "data/evidence_gathering/google_search_results",
+    "faiss_db_embeddings_for_evidence": "data/faiss_db_embeddings_for_evidence",
+    "web_evidence_chunks": "data/evidence_gathering/web_evidence_chunks",
+    # claim evaluation
+    "evaluated_claims_dir": "data/claim_evaluation/claim_verdicts",
+    # reformatted document
+    "fact_checked_document_dir": "data/fact_checked_documents",
+}
+
+PIPELINE_PATHS = {key: Path(value) for key, value in PIPELINE_PATHS.items()}

requirements.in

@@ -0,0 +1,18 @@
+zsvision
+openai
+cohere
+ipdb
+weaviate-client
+bs4
+json5
+fake-useragent
+newspaper3k
+google-api-python-client
+faiss-cpu
+tiktoken
+langchain
+python-dotenv
+langchain-community
+langchain-openai
+pip-tools
+gradio

requirements.txt

@@ -0,0 +1,508 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+aiofiles==23.2.1
+    # via gradio
+aiohttp==3.9.3
+    # via
+    #   cohere
+    #   langchain
+    #   langchain-community
+aiosignal==1.3.1
+    # via aiohttp
+altair==5.2.0
+    # via gradio
+annotated-types==0.6.0
+    # via pydantic
+anyio==4.2.0
+    # via
+    #   httpx
+    #   langchain-core
+    #   openai
+    #   starlette
+asttokens==2.4.1
+    # via stack-data
+attrs==23.2.0
+    # via
+    #   aiohttp
+    #   jsonschema
+    #   referencing
+authlib==1.3.0
+    # via weaviate-client
+backoff==2.2.1
+    # via cohere
+beartype==0.17.2
+    # via zsvision
+beautifulsoup4==4.12.3
+    # via
+    #   bs4
+    #   feedfinder2
+    #   newspaper3k
+bs4==0.0.2
+    # via -r requirements.in
+build==1.0.3
+    # via pip-tools
+cachetools==5.3.2
+    # via google-auth
+certifi==2024.2.2
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+cffi==1.16.0
+    # via cryptography
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via
+    #   nltk
+    #   pip-tools
+    #   typer
+    #   uvicorn
+cohere==4.47
+    # via -r requirements.in
+colorama==0.4.6
+    # via typer
+contourpy==1.2.0
+    # via matplotlib
+cryptography==42.0.2
+    # via authlib
+cssselect==1.2.0
+    # via newspaper3k
+cycler==0.12.1
+    # via matplotlib
+dataclasses-json==0.6.4
+    # via
+    #   langchain
+    #   langchain-community
+decorator==5.1.1
+    # via
+    #   ipdb
+    #   ipython
+distro==1.9.0
+    # via openai
+executing==2.0.1
+    # via stack-data
+faiss-cpu==1.7.4
+    # via -r requirements.in
+fake-useragent==1.4.0
+    # via -r requirements.in
+fastapi==0.109.2
+    # via gradio
+fastavro==1.9.4
+    # via cohere
+feedfinder2==0.0.4
+    # via newspaper3k
+feedparser==6.0.11
+    # via newspaper3k
+ffmpy==0.3.2
+    # via gradio
+filelock==3.13.1
+    # via
+    #   huggingface-hub
+    #   tldextract
+fonttools==4.48.1
+    # via matplotlib
+frozenlist==1.4.1
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2024.2.0
+    # via
+    #   gradio-client
+    #   huggingface-hub
+google-api-core==2.17.0
+    # via google-api-python-client
+google-api-python-client==2.118.0
+    # via -r requirements.in
+google-auth==2.27.0
+    # via
+    #   google-api-core
+    #   google-api-python-client
+    #   google-auth-httplib2
+google-auth-httplib2==0.2.0
+    # via google-api-python-client
+googleapis-common-protos==1.62.0
+    # via google-api-core
+gradio==4.19.0
+    # via -r requirements.in
+gradio-client==0.10.0
+    # via gradio
+grpcio==1.60.1
+    # via
+    #   grpcio-health-checking
+    #   grpcio-tools
+    #   weaviate-client
+grpcio-health-checking==1.60.1
+    # via weaviate-client
+grpcio-tools==1.60.1
+    # via weaviate-client
+h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+h5py==3.10.0
+    # via hickle
+hickle==5.0.2
+    # via zsvision
+httpcore==1.0.2
+    # via httpx
+httplib2==0.22.0
+    # via
+    #   google-api-python-client
+    #   google-auth-httplib2
+httpx==0.26.0
+    # via
+    #   gradio
+    #   gradio-client
+    #   openai
+    #   weaviate-client
+huggingface-hub==0.20.3
+    # via
+    #   gradio
+    #   gradio-client
+humanize==4.9.0
+    # via zsvision
+idna==3.6
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   tldextract
+    #   yarl
+importlib-metadata==6.11.0
+    # via cohere
+importlib-resources==6.1.1
+    # via gradio
+ipdb==0.13.13
+    # via -r requirements.in
+ipython==8.21.0
+    # via ipdb
+jedi==0.19.1
+    # via ipython
+jieba3k==0.35.1
+    # via newspaper3k
+jinja2==3.1.3
+    # via
+    #   altair
+    #   gradio
+joblib==1.3.2
+    # via nltk
+json5==0.9.14
+    # via -r requirements.in
+jsonpatch==1.33
+    # via
+    #   langchain
+    #   langchain-core
+jsonpointer==2.4
+    # via jsonpatch
+jsonschema==4.21.1
+    # via altair
+jsonschema-specifications==2023.12.1
+    # via jsonschema
+kiwisolver==1.4.5
+    # via matplotlib
+langchain==0.1.7
+    # via -r requirements.in
+langchain-community==0.0.20
+    # via
+    #   -r requirements.in
+    #   langchain
+langchain-core==0.1.23
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-openai
+langchain-openai==0.0.6
+    # via -r requirements.in
+langsmith==0.0.87
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+lxml==5.1.0
+    # via newspaper3k
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via
+    #   gradio
+    #   jinja2
+marshmallow==3.20.2
+    # via dataclasses-json
+matplotlib==3.8.2
+    # via
+    #   gradio
+    #   zsvision
+matplotlib-inline==0.1.6
+    # via ipython
+mdurl==0.1.2
+    # via markdown-it-py
+mergedeep==1.3.4
+    # via zsvision
+msgpack==1.0.7
+    # via
+    #   msgpack-numpy
+    #   zsvision
+msgpack-numpy==0.4.8
+    # via zsvision
+multidict==6.0.5
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.0.0
+    # via typing-inspect
+newspaper3k==0.2.8
+    # via -r requirements.in
+nltk==3.8.1
+    # via newspaper3k
+numpy==1.26.4
+    # via
+    #   altair
+    #   contourpy
+    #   gradio
+    #   h5py
+    #   hickle
+    #   langchain
+    #   langchain-community
+    #   langchain-openai
+    #   matplotlib
+    #   msgpack-numpy
+    #   pandas
+    #   scipy
+    #   zsvision
+openai==1.12.0
+    # via
+    #   -r requirements.in
+    #   langchain-openai
+orjson==3.9.14
+    # via gradio
+packaging==23.2
+    # via
+    #   altair
+    #   build
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   langchain-core
+    #   marshmallow
+    #   matplotlib
+pandas==2.2.0
+    # via
+    #   altair
+    #   gradio
+parso==0.8.3
+    # via jedi
+pexpect==4.9.0
+    # via ipython
+pillow==10.2.0
+    # via
+    #   gradio
+    #   matplotlib
+    #   newspaper3k
+pip-tools==7.3.0
+    # via -r requirements.in
+prompt-toolkit==3.0.43
+    # via ipython
+protobuf==4.25.2
+    # via
+    #   google-api-core
+    #   googleapis-common-protos
+    #   grpcio-health-checking
+    #   grpcio-tools
+ptyprocess==0.7.0
+    # via pexpect
+pure-eval==0.2.2
+    # via stack-data
+pyasn1==0.5.1
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.3.0
+    # via google-auth
+pycparser==2.21
+    # via cffi
+pydantic==2.6.1
+    # via
+    #   fastapi
+    #   gradio
+    #   langchain
+    #   langchain-core
+    #   langsmith
+    #   openai
+    #   weaviate-client
+pydantic-core==2.16.2
+    # via pydantic
+pydub==0.25.1
+    # via gradio
+pygments==2.17.2
+    # via
+    #   ipython
+    #   rich
+pyparsing==3.1.1
+    # via
+    #   httplib2
+    #   matplotlib
+pyproject-hooks==1.0.0
+    # via build
+python-dateutil==2.8.2
+    # via
+    #   matplotlib
+    #   newspaper3k
+    #   pandas
+python-dotenv==1.0.1
+    # via -r requirements.in
+python-multipart==0.0.9
+    # via gradio
+pytz==2024.1
+    # via pandas
+pyyaml==6.0.1
+    # via
+    #   gradio
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+    #   newspaper3k
+    #   zsvision
+referencing==0.33.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+regex==2023.12.25
+    # via
+    #   nltk
+    #   tiktoken
+requests==2.31.0
+    # via
+    #   cohere
+    #   feedfinder2
+    #   google-api-core
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+    #   langsmith
+    #   newspaper3k
+    #   requests-file
+    #   tiktoken
+    #   tldextract
+    #   weaviate-client
+requests-file==2.0.0
+    # via tldextract
+rich==13.7.0
+    # via typer
+rpds-py==0.18.0
+    # via
+    #   jsonschema
+    #   referencing
+rsa==4.9
+    # via google-auth
+ruff==0.2.1
+    # via gradio
+scipy==1.12.0
+    # via zsvision
+semantic-version==2.10.0
+    # via gradio
+sgmllib3k==1.0.0
+    # via feedparser
+shellingham==1.5.4
+    # via typer
+six==1.16.0
+    # via
+    #   asttokens
+    #   feedfinder2
+    #   python-dateutil
+sniffio==1.3.0
+    # via
+    #   anyio
+    #   httpx
+    #   openai
+soupsieve==2.5
+    # via beautifulsoup4
+sqlalchemy==2.0.27
+    # via
+    #   langchain
+    #   langchain-community
+stack-data==0.6.3
+    # via ipython
+starlette==0.36.3
+    # via fastapi
+tenacity==8.2.3
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+tiktoken==0.6.0
+    # via
+    #   -r requirements.in
+    #   langchain-openai
+tinysegmenter==0.3
+    # via newspaper3k
+tldextract==5.1.1
+    # via newspaper3k
+tomlkit==0.12.0
+    # via gradio
+toolz==0.12.1
+    # via altair
+tqdm==4.66.2
+    # via
+    #   huggingface-hub
+    #   nltk
+    #   openai
+traitlets==5.14.1
+    # via
+    #   ipython
+    #   matplotlib-inline
+typeguard==4.1.5
+    # via zsvision
+typer[all]==0.9.0
+    # via gradio
+typing-extensions==4.9.0
+    # via
+    #   fastapi
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   openai
+    #   pydantic
+    #   pydantic-core
+    #   sqlalchemy
+    #   typeguard
+    #   typer
+    #   typing-inspect
+typing-inspect==0.9.0
+    # via dataclasses-json
+tzdata==2024.1
+    # via pandas
+uritemplate==4.1.1
+    # via google-api-python-client
+urllib3==2.2.0
+    # via
+    #   cohere
+    #   requests
+uvicorn==0.27.1
+    # via gradio
+validators==0.22.0
+    # via weaviate-client
+wcwidth==0.2.13
+    # via prompt-toolkit
+weaviate-client==4.4.4
+    # via -r requirements.in
+websockets==11.0.3
+    # via gradio-client
+wheel==0.42.0
+    # via pip-tools
+yarl==1.9.4
+    # via aiohttp
+zipp==3.17.0
+    # via importlib-metadata
+zsvision==0.7.12
+    # via -r requirements.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools

run_pipeline.py

@@ -0,0 +1,134 @@
+from step1_api_claim_extractor import ClaimExtractor
+from step2_api_fix_passage_anchors import FixAnchors
+from step3_api_identify_objective_claims import ClassifyClaims
+from step41_api_fetch_cohere_wikipedia_evidence import CohereEvidence
+from step42_api_fetch_google_search_evidence import GoogleEvidence
+from step5_api_embed_search_results import EmbedResults
+from step6_api_claims_to_evidence import ClaimToEvidence
+from step7_api_check_claims_against_evidence import CheckClaimAgainstEvidence
+from step8_api_format_fact_checked_document import FormatDocument
+
+import argparse
+import json
+import os
+import copy
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def get_fact_checked(text_input, model="gpt-3.5-turbo", mode="slow"):
+    text_input = text_input.strip()
+
+    results = {}
+
+    # STEP1
+    print("Step1: Extracting claims")
+    step1 = ClaimExtractor(model=model)
+    step1_json = step1.extract_claims(text_input)
+    results["step1_claims"] = copy.deepcopy(step1_json)
+
+    # STEP2
+    print("Step2: Anchoring claims")
+    try:
+        step2 = FixAnchors(model=model)
+        step2_json = step2.fix_passage_anchors(step1_json, text_input)
+    except:
+        if model != "gpt-4":
+            print("Step2 failed with gpt-3.5, trying with gpt-4!")
+            step2 = FixAnchors(model="gpt-4")
+            step2_json = step2.fix_passage_anchors(step1_json, text_input)
+    results["step2_anchored_claims"] = copy.deepcopy(step2_json)
+
+    # STEP3
+    print("Step3: Classifying claims")
+    step3 = ClassifyClaims(model=model)
+    step3_json = step3.classify_claims(step2_json)
+    step3_filter = step3.filter_to_objective_claims(step3_json)
+    results["step3_classify_claims"] = copy.deepcopy(step3_json)
+    results["step3_objective_claims"] = copy.deepcopy(step3_filter)
+
+    if len(step3_filter) == 0:
+        return {"fact_checked_md": "No objective claims found!"}
+
+    # STEP4.1
+    print("Step4.1: Gathering evidence")
+    step4_cohere = CohereEvidence()
+    step4_json_cohere = (
+        step4_cohere.fetch_cohere_semantic_search_results_to_gather_evidence(
+            step3_filter
+        )
+    )
+    results["step41_cohere_evidence"] = copy.deepcopy(step4_json_cohere)
+
+    # STEP4.2
+    print("Step4.2: Gathering evidence")
+    step4_json_google = None
+    if mode == "slow":
+        step4_json_google = ""
+        try:
+            step4_google = GoogleEvidence(model=model)
+            step4_json_google = step4_google.fetch_search_results_to_gather_evidence(
+                step3_filter
+            )
+        except Exception as e:
+            print(f"Google search failed: {e}")
+            pass
+        results["step42_google_evidence"] = copy.deepcopy(step4_json_google)
+
+    embedding_model = "text-embedding-ada-002"
+    text_embedding_chunk_size = 500
+
+    srcs = [step4_json_cohere]
+    if step4_json_google:
+        srcs.append(step4_json_google)
+
+    # STEP 5
+    print("Step5: Embedding evidence")
+    step5 = EmbedResults(
+        embedding_model=embedding_model,
+        text_embedding_chunk_size=text_embedding_chunk_size,
+    )
+    faiss_db = step5.embed_for_uuid(srcs)
+
+    # STEP 6
+    print("Step6: Linking claims to evidence")
+    step6 = ClaimToEvidence()
+    step6_json = step6.link_claims_to_evidence(step3_filter, faiss_db)
+    results["step6_claim_to_evidence"] = copy.deepcopy(step6_json)
+
+    # STEP 7
+    print("Step7: Checking claims against evidence")
+    step7 = CheckClaimAgainstEvidence(model=model)
+    step7_json = step7.check_claims_against_evidence(step6_json)
+    results["step7_evaluated_claims"] = copy.deepcopy(step7_json)
+
+    # STEP 8
+    print("Step8: Formatting")
+    if mode == "slow":
+        step8 = FormatDocument(model=model, footnote_style="verbose")
+        step8_md = step8.reformat_document_to_include_claims(
+            text_input, step7_json, footnote_style="verbose"
+        )
+        step8_md_terse = step8.reformat_document_to_include_claims(
+            text_input, step7_json, footnote_style="terse"
+        )
+
+        results["fact_checked_md"] = copy.deepcopy(step8_md)
+        results["fact_checked_terse"] = copy.deepcopy(step8_md_terse)
+    return results
+
+
+def main(args):
+    with open(args.file, "r") as f:
+        text = f.read()
+    out = get_fact_checked(text, mode="slow", model=args.model)
+    print(out["fact_checked_md"])
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Process a file.")
+    parser.add_argument("--file", type=str, help="File to process", required=True)
+    parser.add_argument("--model", type=str, help="Model to use", required=True)
+    args = parser.parse_args()
+    main(args)

step1_api_claim_extractor.py

@@ -0,0 +1,164 @@
+import argparse
+import json
+import multiprocessing as mp
+from zsvision.zs_multiproc import starmap_with_kwargs
+from zsvision.zs_utils import BlockTimer
+from text_utils import is_unique_verbatim_quote, parse_passage_quote_and_claim
+from llm_api_utils import (
+    call_openai_with_exponetial_backoff,
+    estimate_cost_of_text_generation_api_call,
+    init_openai_with_api_key,
+)
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+class ClaimExtractor:
+    def __init__(
+        self,
+        temperature=0,
+        model="gpt-3.5-turbo",
+        filter_str="",
+        processes=1,
+        refresh=False,
+    ):
+        """Initializes ClaimExtractor with the provided arguments"""
+        self.temperature = temperature
+        self.model = model
+        self.filter_str = filter_str
+        self.processes = processes
+        self.refresh = refresh
+
+    def extract_claims_from_passage(
+        self,
+        idx: int,
+        total: int,
+        passage: str,
+    ):
+        init_openai_with_api_key()
+        print(f"Processing passage {idx + 1} of {total}")
+        prompt = f"""\
+Task:
+Enumerate all the discrete factual claims or logical assertions stated in the passage that follows the dashed horizontal line below. \
+To allow the claims to be linked to the passage, use the format: `VERBATIM_PASSAGE_QUOTE_FOR_CLAIM: <verbatim passage quote for claim>, CLAIM: <claim>` on each line. \
+The <verbatim passage quote for claim> must be A SINGLE UNEDITED SUBSTRING from the passage that uniquely identifies the claim. \
+The <verbatim passage quote for claim> must carefully preserve all punctuation and clauses from the original passage. \
+This text will be used in the final national exam.
+
+----------
+Here is an example passage, together with the verbatim passage quotes and claims that should be extracted from it:
+
+Passage:
+Immanuel Kant was born in 1724 into a modest, devoutly religious family, with his father working as a saddle-maker. \
+He was one of nine children, but only five, including Kant, survived to adulthood. \
+His upbringing was steeped in the Pietist tradition, emphasizing intense religious devotion, a literal interpretation of the Bible, and a strong focus on personal morality. \
+Kant attended the University of Königsberg, studying various subjects, including theology, metaphysics, and natural science. \
+After completing his studies, Kant worked as a private tutor for nine years before returning to the University of Königsberg as a lecturer in 1755. \
+In his works Groundwork of the Metaphysics of Morals (1785) and Critique of Practical Reason (1788), Kant argues that morality is not contingent upon personal desires or cultural norms. \
+
+
+Extracted source phrases and claims:
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] Immanuel Kant was born in 1724 into a modest, devoutly religious family [CLAIM] Immanuel Kant was born in 1724.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] Immanuel Kant was born in 1724 into a modest, devoutly religious family [CLAIM] Immanuel Kant was born into a modest family.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] Immanuel Kant was born in 1724 into a modest, devoutly religious family [CLAIM] Immanuel Kant was born into a devoutly religious family.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] with his father working as a saddle-maker [CLAIM] Immnauel Kant's father worked as a saddle-maker.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] He was one of nine children [CLAIM] Immanuel Kant was one of nine children.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] but only five, including Kant survived to adulthood [CLAIM] Only five of Immanuel Kant's parents' children survived to adulthood.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] His upbringing was steeped in the Pietist tradition [CLAIM] Immanuel Kant's upbringing was steeped in the Pietist tradition.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] emphasizing intense religious devotion [CLAIM] Immanuel Kant's upbringing emphasized intense religious devotion.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] a literal interpretation of the Bible [CLAIM] Immanuel Kant's upbringing emphasized a literal interpretation of the Bible.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] a strong focus on personal morality [CLAIM] Immanuel Kant's upbringing emphasized a strong focus on personal morality.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] Kant attended the University of Königsberg [CLAIM] Immanuel Kant attended the University of Königsberg.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] studying various subjects, including theology, metaphysics, and natural science [CLAIM] Immanuel Kant studied theology.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] studying various subjects, including theology, metaphysics, and natural science [CLAIM] Immanuel Kant studied metaphysics.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] studying various subjects, including theology, metaphysics, and natural science [CLAIM] Immanuel Kant studied natural science.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] After completing his studies [CLAIM] Immanuel Kant completed his studies.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] After completing his studies, Kant worked as a private tutor for nine years [CLAIM] After completing his studies, Immanuel Kant worked as a private tutor.
+[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] before returning to the University of Königsberg as a lecturer in 1755 [CLAIM] Immanuel Kant returned to the University of Königsberg as a lecturer in 1755.
+
+----------
+Passage:
+{passage}
+
+Extracted source phrases and claims:\
+"""
+        persona = "You are a careful research assistant who helps with fact-checking and editing informative articles."
+        system_message = {"role": "system", "content": persona}
+        user_message = {"role": "user", "content": prompt}
+        messages = [system_message, user_message]
+
+        with BlockTimer(f"Using OpenAI API to extract claims with {self.model}"):
+            response = call_openai_with_exponetial_backoff(
+                model=self.model,
+                temperature=self.temperature,
+                messages=messages,
+            )
+
+        cost = estimate_cost_of_text_generation_api_call(
+            model=self.model, response=response, verbose=True
+        )
+        content = response.choices[0].message.content
+        content = content.strip()
+        quotes_and_claims = content.split("\n")
+
+        parsed_claims = []
+        for quote_and_claim in quotes_and_claims:
+            quote_and_claim = quote_and_claim.strip()
+            if "[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM]" not in quote_and_claim:
+                quote_and_claim = quote_and_claim.replace(
+                    "VERBATIM_PASSAGE_QUOTE_FOR_CLAIM: ",
+                    "[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM]: ",
+                )
+            if "[CLAIM]" not in quote_and_claim:
+                quote_and_claim = quote_and_claim.replace(" CLAIM:", " [CLAIM]:")
+
+            if "[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM]" not in quote_and_claim:
+                continue
+            quote_and_claim = quote_and_claim.strip()
+            parsed = parse_passage_quote_and_claim(quote_and_claim)
+            is_unique_and_verbatim = is_unique_verbatim_quote(
+                verbatim_quote=parsed["verbatim_quote"], original_passage=passage
+            )
+            parsed["is_unique_and_verbatim"] = is_unique_and_verbatim
+            parsed_claims.append(parsed)
+
+        return {"claims": parsed_claims, "cost": cost}
+
+    def extract_claims(self, text_input):
+        """
+        Extracts claims from text_input and return the extracted claims in a json file
+        """
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+        docs = text_splitter.create_documents([text_input])
+        print(f"Split text into {len(docs)} documents")
+        all_claims = []
+
+        kwarg_list = []
+        for idx, doc in enumerate(docs):
+            # remove newlines from the passage to avoid a confusing prompt format
+            passage = doc.page_content.replace("\n", " ")
+            kwarg_list.append(
+                {
+                    "idx": idx,
+                    "total": len(docs),
+                    "passage": passage,
+                }
+            )
+
+        if self.processes == 1:
+            results = []
+            for kwargs in kwarg_list:
+                results.append(self.extract_claims_from_passage(**kwargs))
+        else:  # multiprocess
+            func = self.extract_claims_from_passage
+            with mp.Pool(processes=self.processes) as pool:
+                results = starmap_with_kwargs(
+                    pool=pool, func=func, kwargs_iter=kwarg_list
+                )
+
+        cost = sum([result["cost"] for result in results])
+        all_claims = []
+        for result in results:
+            all_claims.extend(result["claims"])
+
+        print(f"Returning {len(all_claims)} claims (cost: {cost} USD)")
+        return all_claims

step2_api_fix_passage_anchors.py

@@ -0,0 +1,165 @@
+import argparse
+import json
+import multiprocessing as mp
+from zsvision.zs_multiproc import starmap_with_kwargs
+from zsvision.zs_utils import BlockTimer
+from text_utils import is_unique_verbatim_quote, parse_passage_quote_and_claim
+from llm_api_utils import (
+    call_openai_with_exponetial_backoff,
+    estimate_cost_of_text_generation_api_call,
+    init_openai_with_api_key,
+)
+
+
+class FixAnchors:
+    def __init__(
+        self,
+        temperature=0,
+        model="gpt-3.5-turbo",
+        filter_str="",
+        processes=8,
+        refresh=False,
+    ):
+        self.temperature = temperature
+        self.model = model
+        self.filter_str = filter_str
+        self.processes = processes
+        self.refresh = refresh
+
+    def fix_passage_anchor(
+        self,
+        idx: int,
+        total: int,
+        original_passage: str,
+        claim_with_metadata: dict,
+    ):
+        init_openai_with_api_key()
+        print(f"Processing claim with metadata {idx + 1} of {total}")
+        # we remove newlines
+        original_passage = original_passage.replace("\n", " ")
+        assert not claim_with_metadata[
+            "is_unique_and_verbatim"
+        ], "We should only fix broken passage anchors"
+
+        prompt = f"""\
+Task:
+A co-worker was tasked with identifying a unique, verbatim quote from a passage that underpins a particular claim. \
+Unfortunately, they made a mistake and the quote they identified is not unique and verbatim. \
+Your task is to fix their quote so that it is both verbatim and unique.
+
+-----
+Here is an example passage, together with the claim and the erroneous quote.
+
+Passage:
+In 1940, she was interned in a French camp as an enemy alien, but managed to escape and eventually make her way to the United States in 1941.  \
+Arendt's experiences during this time would deeply influence her work on totalitarianism and human rights. \
+In New York, she began to immerse herself in academic life, working as an editor, journalist, and lecturer. \
+Her first major work, *The Origins of Totalitarianism*, published in 1951, explored the common roots of Nazism and Stalinism, and established her as a significant voice in political philosophy. \
+## A Life Of Controversial, Influential Works  \
+Throughout her career, Arendt wrote a number of seminal, and controversial, works. *The Human Condition* (1958) examined the role of politics in modern societies and introduced the concept of "the public realm" – the space where individuals act and participate in political life. \
+This exploration of freedom and action would become a recurring theme in her writings.  \
+Her 1963 publication, *Eichmann in Jerusalem: A Report on the Banality of Evil*, based on her coverage of Adolf Eichmann's trial, ignited significant controversy. \
+Arendt argued that Eichmann, a key architect of the Holocaust, was not a monster but rather an ordinary bureaucrat who unquestioningly followed orders. \
+The idea of the "banality of evil" continues to influence discussions on the nature of evil and moral responsibility.  \
+Arendt's later works, such as *On Revolution* (1963) and *Between Past and Future* (1968), sought to further unravel the complexities of power, authority, and rebellion. \
+Her writings on these subjects continue to resonate with present-day political struggles, as well as with the works of other philosophers like [Immanuel Kant](/philosophy/2023-immanuel-kant-life-and-work) and [Edmund Husserl](/philosophy/2023-edmund-husserl-his-life-and-work).  \
+
+Claim:
+*The Origins of Totalitarianism* established Arendt as a significant voice in political philosophy.
+
+Initial attempt at a unique and verbatim quote:
+[The Origins of Totalitarianism] established her as a significant voice in political philosophy.
+
+Correct (unique and verbatim) quote:
+Her first major work, *The Origins of Totalitarianism*, published in 1951, explored the common roots of Nazism and Stalinism, and established her as a significant voice in political philosophy.
+-----
+Passage:
+{original_passage}
+
+Claim:
+{claim_with_metadata["claim"]}
+
+Initial attempt at a unique verbatim quote:
+{claim_with_metadata["verbatim_quote"]}
+
+Correct (unique and verbatim) quote:\
+"""
+        persona = "You are a careful research assistant who helps with fact-checking and editing informative articles."
+        system_message = {"role": "system", "content": persona}
+        user_message = {"role": "user", "content": prompt}
+        messages = [system_message, user_message]
+
+        with BlockTimer(f"Using OpenAI API to extract claims with {self.model}"):
+            response = call_openai_with_exponetial_backoff(
+                model=self.model,
+                temperature=self.temperature,
+                messages=messages,
+            )
+
+        cost = estimate_cost_of_text_generation_api_call(
+            model=self.model, response=response, verbose=True
+        )
+        content = response.choices[0].message.content
+        verbatim_quote = content.rstrip()
+        is_unique_and_verbatim = is_unique_verbatim_quote(
+            verbatim_quote=verbatim_quote, original_passage=original_passage
+        )
+        assert (
+            is_unique_and_verbatim
+        ), f"Failed to fix passage anchor: {claim_with_metadata['verbatim_quote']} was updated to {verbatim_quote} but is not unique and verbatim"
+
+        claim_with_metadata["verbatim_quote"] = verbatim_quote
+        return {"claim_with_metadata": claim_with_metadata, "cost": cost}
+
+    def fix_passage_anchors(self, claims_with_metadata, original_passage: str):
+        kwarg_list = []
+        valid_claims_with_metadata = []
+        invalid_claims_with_metadata = []
+        for idx, claim_with_metadata in enumerate(claims_with_metadata):
+            # remove newlines from the passage to avoid a confusing prompt format
+            if not claim_with_metadata["is_unique_and_verbatim"]:
+                invalid_claims_with_metadata.append(claim_with_metadata)
+            else:
+                valid_claims_with_metadata.append(claim_with_metadata)
+
+        for idx, claim_with_metadata in enumerate(invalid_claims_with_metadata):
+            kwarg_list.append(
+                {
+                    "idx": idx,
+                    "total": len(invalid_claims_with_metadata),
+                    "claim_with_metadata": claim_with_metadata,
+                    "original_passage": original_passage,
+                }
+            )
+
+        if self.processes == 1:
+            results = []
+            for kwargs in kwarg_list:
+                try:
+                    results.append(self.fix_passage_anchor(**kwargs))
+                except Exception as e:
+                    print(f"Exception in step2: {e}, model: {self.model}")
+                    print("Skipping this claim!")
+                    if self.model == "gpt-4":
+                        pass
+                    else:
+                        raise e
+        else:  # multiprocess
+            func = self.fix_passage_anchor
+            with mp.Pool(processes=self.processes) as pool:
+                results = starmap_with_kwargs(
+                    pool=pool, func=func, kwargs_iter=kwarg_list
+                )
+
+        cost = sum([result["cost"] for result in results])
+        for result in results:
+            valid_claims_with_metadata.append(result["claim_with_metadata"])
+
+        # remove the is_unique_and_verbatim field (no longer needed)
+        for claim_with_metadata in valid_claims_with_metadata:
+            del claim_with_metadata["is_unique_and_verbatim"]
+
+        print(
+            f"Returning {len(valid_claims_with_metadata)} claims with metadat (cost: {cost} USD)"
+        )
+        return valid_claims_with_metadata

step3_api_identify_objective_claims.py

@@ -0,0 +1,185 @@
+import json
+import argparse
+import multiprocessing as mp
+from zsvision.zs_multiproc import starmap_with_kwargs
+from typing import List, Dict
+import numpy as np
+from zsvision.zs_utils import BlockTimer
+from llm_api_utils import (
+    call_openai_with_exponetial_backoff,
+    estimate_cost_of_text_generation_api_call,
+    init_openai_with_api_key,
+)
+import random
+
+
+class ClassifyClaims:
+    def __init__(
+        self,
+        temperature=0,
+        model="gpt-3.5-turbo",
+        max_claims_per_api_call=10,
+        processes=8,
+        filter_str="",
+        refresh=False,
+    ):
+        self.temperature = temperature
+        self.model = model
+        self.max_claims_per_api_call = max_claims_per_api_call
+        self.processes = processes
+        self.filter_str = filter_str
+        self.refresh = refresh
+        self.objective_claims_file = "objective_claims.txt"
+        self.subjective_claims_file = "subjective_claims.txt"
+
+    def parse_classification_label(self, text: str) -> str:
+        raw = text.strip()
+        if raw.endswith("[objective]"):
+            label = "objective"
+        elif raw.endswith("[subjective]"):
+            label = "subjective"
+        else:
+            raise ValueError(f"Invalid label: {raw}")
+        return label
+
+    def read_file(self, file_name):
+        with open(file_name, "r") as f:
+            lines = []
+            for line in f:
+                parsed_line = line.strip()
+                lines.append(parsed_line)
+        return lines
+
+    def create_few_shot_learning_prompt(self) -> str:
+        objective_list = self.read_file(self.objective_claims_file)
+        subjective_list = self.read_file(self.subjective_claims_file)
+        merged_list = list(
+            zip(objective_list, ["[objective]"] * len(objective_list))
+        ) + list(zip(subjective_list, ["[subjective]"] * len(subjective_list)))
+
+        # Randomizing the merged list with a specific seed
+        seed = 1234
+        random.seed(seed)
+        random.shuffle(merged_list)
+        prompt = "Claims:\n"
+        for claim, _ in merged_list:
+            prompt += claim + "\n"
+        prompt += "\nClassifications:\n"
+        for claim, classif in merged_list:
+            prompt += claim + " " + classif + "\n"
+        return prompt
+
+    def classify_claim_batch(
+        self,
+        idx: int,
+        total: int,
+        claims_and_sources_batch: List[Dict[str, str]],
+    ):
+        print(
+            f"Processing batch {idx+1} of {total} (containing {len(claims_and_sources_batch)} claims)"
+        )
+
+        claim_str = "\n".join([claim["claim"] for claim in claims_and_sources_batch])
+        num_batch_claims = len(claims_and_sources_batch)
+        few_shot = self.create_few_shot_learning_prompt()
+        prompt = f"""\
+Objective claims can be verified based on factual data (such as those that could be verified by \
+referencing an encyclopedia), whereas subjective claims involve a personal interpretation of \
+the data and are more open to debate. \
+For each of the following claims given below the dashed horizontal line, classify them as \
+[subjective] or [objective] by suffixing the claim with the appropriate label. OUTPUT ONLY the class, either subjective or objective for each claim!
+
+Here are some examples:
+
+{few_shot}
+----------
+Claims:
+{claim_str}
+
+Classifications:\
+"""
+        persona = "You are a careful research assistant who helps with fact-checking and editing informative articles."
+        system_message = {"role": "system", "content": persona}
+        user_message = {"role": "user", "content": prompt}
+        messages = [system_message, user_message]
+
+        with BlockTimer(f"Using OpenAI API to extract claims with {self.model}"):
+            response = call_openai_with_exponetial_backoff(
+                model=self.model,
+                temperature=self.temperature,
+                messages=messages,
+            )
+
+        cost = estimate_cost_of_text_generation_api_call(
+            model=self.model, response=response, verbose=True
+        )
+
+        proposed_classified_claims = response.choices[0].message.content
+        batch_classified_claims = proposed_classified_claims.split("\n")
+
+        content = response.choices[0].message.content
+        batch_classified_claims = content.split("\n")
+        assert (
+            len(batch_classified_claims) == num_batch_claims
+        ), f"Expected {num_batch_claims} claims, but got {len(batch_classified_claims)}"
+        print(f"Generated {len(batch_classified_claims)} claims (cost: {cost:.4f} USD)")
+
+        claims_with_labels = []
+        for claim_and_source, classified_claim in zip(
+            claims_and_sources_batch, batch_classified_claims
+        ):
+            claim_label = self.parse_classification_label(classified_claim)
+            claim_and_source["label"] = claim_label
+            claims_with_labels.append(claim_and_source)
+        return {"claims_with_labels": claims_with_labels, "cost": cost}
+
+    def classify_claims(self, claims_and_sources):
+        """
+        Classify claims as being either subjective or objective, and write the results to a file.
+        """
+        init_openai_with_api_key()
+        num_claims = len(claims_and_sources)
+
+        # we limit the number of claims per api call (otherwise GPT-4 can choke)
+        num_batches = int(np.ceil(num_claims / self.max_claims_per_api_call))
+        claims_and_sources_batches = [
+            batch.tolist() for batch in np.array_split(claims_and_sources, num_batches)
+        ]
+
+        kwarg_list = []
+        for idx, claims_and_sources_batch in enumerate(claims_and_sources_batches):
+            # remove newlines from the passage to avoid a confusing prompt format
+            kwarg_list.append(
+                {
+                    "idx": idx,
+                    "total": len(claims_and_sources_batches),
+                    "claims_and_sources_batch": claims_and_sources_batch,
+                }
+            )
+
+        if self.processes == 1:
+            batch_results = []
+            for kwargs in kwarg_list:
+                batch_results.append(self.classify_claim_batch(**kwargs))
+        else:  # multiprocess
+            func = self.classify_claim_batch
+            with mp.Pool(processes=self.processes) as pool:
+                batch_results = starmap_with_kwargs(
+                    pool=pool, func=func, kwargs_iter=kwarg_list
+                )
+
+        cost = sum([result["cost"] for result in batch_results])
+        labelled_claims = []
+        for batch in batch_results:
+            labelled_claims.extend(batch["claims_with_labels"])
+
+        print(f"Returning {len(labelled_claims)} claims (cost: {cost} USD)")
+        return labelled_claims
+
+    def filter_to_objective_claims(self, claims):
+        """Filter claims to only those that are objective."""
+
+        objective_claims = [claim for claim in claims if claim["label"] == "objective"]
+
+        print(f"Returning {len(objective_claims)} objective claims")
+        return objective_claims

step41_api_fetch_cohere_wikipedia_evidence.py

@@ -0,0 +1,97 @@
+import argparse
+import json
+import multiprocessing as mp
+from datetime import datetime
+import time
+from zsvision.zs_multiproc import starmap_with_kwargs
+import weaviate
+import os
+
+
+class CohereEvidence:
+    def __init__(self, processes=8, filter_str="", refresh=False):
+        self.processes = processes
+        self.filter_str = filter_str
+        self.refresh = refresh
+
+    def semantic_search(self, query, client, results_lang=""):
+        """
+        Query the vectors database and return the top results.
+        """
+
+        nearText = {"concepts": [query]}
+        properties = ["text", "title", "url", "views", "lang", "_additional {distance}"]
+
+        # To filter by language
+        if results_lang != "":
+            where_filter = {
+                "path": ["lang"],
+                "operator": "Equal",
+                "valueString": results_lang,
+            }
+            response = (
+                client.query.get("Articles", properties)
+                .with_where(where_filter)
+                .with_near_text(nearText)
+                .with_limit(5)
+                .do()
+            )
+
+        # Search all languages
+        else:
+            response = (
+                client.query.get("Articles", properties)
+                .with_near_text(nearText)
+                .with_limit(5)
+                .do()
+            )
+
+        result = response["data"]["Get"]["Articles"]
+
+        return result
+
+    def fetch_cohere_semantic_search_results_to_gather_evidence(
+        self,
+        queryset: dict,
+    ):
+        """
+        Generate a search query that can be used to verify a claim.
+        """
+        # 10M wiki embeddings (1M in English)
+        weaviate_api_key = os.environ.get("WEAVIATE_API_KEY")
+
+        cohere_api_key = os.environ.get("COHERE_API_KEY")
+
+        client = weaviate.Client(
+            url="https://cohere-demo.weaviate.network/",
+            auth_client_secret=weaviate.auth.AuthApiKey(
+                api_key=weaviate_api_key
+            ),  # Replace w/ your Weaviate instance API key
+            additional_headers={
+                "X-Cohere-Api-Key": cohere_api_key  # Replace with your inference API key
+            },
+        )
+
+        while not client.is_ready():
+            print(f"Waiting for client to be ready")
+            time.sleep(1)
+
+        for item in queryset:
+            results = self.semantic_search(
+                item["claim"], client=client, results_lang="en"
+            )
+            # rename "url" to "link" to be consistent with google results
+            reformatted_results = []
+            for result in results:
+                result["link"] = result.pop("url")
+                reformatted_results.append(result)
+            item["search_results"] = reformatted_results
+
+        # update the queryset with new information
+        date_str = datetime.now().strftime("%Y-%m-%d")
+        results = {
+            "documents": queryset,
+            "dates": {"results_fetched_from_wikipedia_1M_with_cohere-22-12": date_str},
+        }
+        print(f"Returning Cohere Wikipedia paragraph for {len(queryset)} queries")
+        return results

step42_api_fetch_google_search_evidence.py

@@ -0,0 +1,145 @@
+import requests
+from bs4 import BeautifulSoup
+from zsvision.zs_utils import BlockTimer
+import json
+import json5
+import argparse
+import multiprocessing as mp
+from zsvision.zs_multiproc import starmap_with_kwargs
+from datetime import datetime
+import urllib.robotparser
+import urllib.parse
+from urllib.parse import urlunparse
+from utils import get_google_search_results
+
+import time
+from random import randint
+from fake_useragent import UserAgent
+from newspaper import Article, Config
+
+
+class GoogleEvidence:
+    def __init__(
+        self,
+        model="gpt-3.5-turbo",
+        limit=0,
+        refresh=False,
+        num_search_results_to_keep=3,
+        filter_str="",
+        processes=8,
+    ):
+        self.model = model
+        self.limit = limit
+        self.refresh = refresh
+        self.num_search_results_to_keep = num_search_results_to_keep
+        self.filter_str = filter_str
+        self.processes = processes
+
+    def can_index(self, url, user_agent_name):
+        rp = urllib.robotparser.RobotFileParser()
+        robots_url = f"{url.scheme}://{url.netloc}/robots.txt"
+
+        headers = {
+            "User-Agent": user_agent_name,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+        }
+
+        try:
+            req = urllib.request.Request(robots_url, headers=headers)
+            with urllib.request.urlopen(req) as response:
+                rp.parse(response.read().decode("utf-8").splitlines())
+
+            ok_to_index = rp.can_fetch(user_agent_name, url.geturl())
+        except urllib.error.URLError:
+            # If there is no robots.txt or there is an error accessing it, assume it's okay to index
+            ok_to_index = True
+        except Exception as e:
+            print(f"An unexpected error occurred in step42: {e}")
+            # going the safe route
+            ok_to_index = False
+        return ok_to_index
+
+    def fetch_search_results_to_gather_evidence(
+        self,
+        queryset: dict,
+    ):
+        user_agent = UserAgent()
+        config = Config()
+        config.fetch_images = False
+
+        user_agent_name = "FiltirBot/1.0 (+https://filtir.com/filtirbot-info)"
+
+        headers = {
+            "User-Agent": user_agent_name,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+        }
+
+        # we assume some sites won't permit indexing, so we'll skip these
+        num_results = self.num_search_results_to_keep + 5
+        results = {}
+
+        print(f"Found {len(queryset)} claims to fetch search results for")
+
+        for queryset_idx, item in enumerate(queryset):
+            with BlockTimer(
+                f"Fetching search results from Google {queryset_idx + 1}/{len(queryset)}"
+            ):
+                search_results = get_google_search_results(
+                    query_str=item["claim"], num_results=num_results
+                )
+
+            if search_results == [{"Result": "No good Google Search Result was found"}]:
+                item["search_results"] = []
+                continue
+
+            parsed_results = []
+            for search_result in search_results:
+                if not self.can_index(
+                    urllib.parse.urlparse(search_result["link"]),
+                    user_agent_name=user_agent_name,
+                ):
+                    print(
+                        f"Skipping {search_result['link']} because it doesn't permit indexing"
+                    )
+                    continue
+                try:
+                    config.browser_user_agent = user_agent.random
+                    article = Article(
+                        search_result["link"], language="en", config=config
+                    )
+                    article.download()
+                    article.parse()
+                    text = article.text
+                except Exception as e:
+                    print(f"Error parsing article: {e}, trying with requests.get...")
+                    try:
+                        response = requests.get(
+                            search_result["link"], timeout=15, headers=headers
+                        )
+                        html = response.text
+                        soup = BeautifulSoup(html, features="html.parser")
+                        text = soup.get_text()
+                    except Exception as exception:
+                        print(f"Error parsing article: {exception}, skipping")
+                        continue
+
+                search_result["text"] = text
+                parsed_results.append(search_result)
+                if len(parsed_results) == self.num_search_results_to_keep:
+                    break
+            item["search_results"] = parsed_results
+
+        # update the queryset with new information
+        date_str = datetime.now().strftime("%Y-%m-%d")
+        results = {"documents": queryset, "dates": {"search_results_fetched": date_str}}
+
+        print(f"Returning web pages for search results for {len(queryset)} queries")
+        return results

step5_api_embed_search_results.py

@@ -0,0 +1,290 @@
+import faiss
+import shutil
+from beartype import beartype
+import numpy as np
+import json
+import argparse
+from zsvision.zs_utils import BlockTimer
+import tiktoken
+from pathlib import Path
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from pipeline_paths import PIPELINE_PATHS
+from llm_api_utils import (
+    init_openai_with_api_key,
+    EMBEDDING_DIMENSIONS,
+    PRICE_PER_1K_TOKENS,
+)
+from langchain_community.vectorstores import FAISS
+from langchain_openai import OpenAIEmbeddings
+from langchain.docstore.in_memory import InMemoryDocstore
+
+
+class EmbedResults:
+    def __init__(
+        self,
+        embedding_model="ada",
+        limit=0,
+        refresh=False,
+        refresh_faiss_db=False,
+        text_embedding_chunk_size=500,
+        filter_str="",
+    ):
+        self.embedding_model = embedding_model
+        self.limit = limit
+        self.refresh = refresh
+        self.refresh_faiss_db = refresh_faiss_db
+        self.text_embedding_chunk_size = text_embedding_chunk_size
+        self.filter_str = filter_str
+
+    @beartype
+    def compute_embeddings_from_chunks(
+        self, embedding_function: OpenAIEmbeddings, metadatas: list, faiss_db
+    ):
+        doc_chunks = []
+        metadatas_without_chunks = []
+        for metadata in metadatas:
+            doc_chunk = metadata.pop("doc_chunk")
+            doc_chunks.append(doc_chunk)
+            metadatas_without_chunks.append(metadata)
+
+        with BlockTimer(f"Embedding {len(metadatas)} fragments"):
+            embeddings = embedding_function.embed_documents(doc_chunks)
+            # account for name mangling in Python
+            faiss_db._FAISS__add(doc_chunks, embeddings, metadatas_without_chunks)
+
+        return faiss_db
+
+    @beartype
+    def parse_date_of_fetching(self, data: dict) -> str:
+        evidence_keys = {
+            "search_results_fetched",
+            "results_fetched_from_wikipedia_1M_with_cohere-22-12",
+        }
+        for key in evidence_keys:
+            if key in data["dates"]:
+                evidence_fetched_date = data["dates"][key]
+                return evidence_fetched_date
+        raise ValueError(f"Could not find evidence fetched date in {data['dates']}")
+
+    def embed_for_uuid(self, srcs):
+        init_openai_with_api_key()
+
+        embedding_function = OpenAIEmbeddings()
+
+        index = faiss.IndexFlatL2(EMBEDDING_DIMENSIONS[self.embedding_model])
+        docstore = InMemoryDocstore({})
+        index_to_docstore_id = {}
+        faiss_db = FAISS(
+            embedding_function=embedding_function.embed_query,
+            index=index,
+            docstore=docstore,
+            index_to_docstore_id=index_to_docstore_id,
+        )
+
+        already_embedded_chunks = {
+            doc.metadata["chunk_tag"] for doc in faiss_db.docstore._dict.values()
+        }
+
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.text_embedding_chunk_size,
+            chunk_overlap=0,
+        )
+
+        kwarg_list = []
+        seen_links = set()
+        metadatas = []
+        total_chunks = 0
+        chunks_to_embed = 0
+        chunks_to_skip = 0
+
+        for data in srcs:
+            evidence_fetched_date = self.parse_date_of_fetching(data)
+
+            for document in data["documents"]:
+                for search_result in document["search_results"]:
+                    # Don't embed the same link twice
+                    if search_result["link"] in seen_links:
+                        continue
+                    seen_links.add(search_result["link"])
+
+                    doc_chunks = [
+                        doc.page_content
+                        for doc in splitter.create_documents([search_result["text"]])
+                    ]
+                    chunk_tags = [
+                        f"{search_result['link']}-chunk-{idx}-chunk_sz-{self.text_embedding_chunk_size}"
+                        for idx in range(len(doc_chunks))
+                    ]
+                    for doc_chunk, chunk_tag in zip(doc_chunks, chunk_tags):
+                        if chunk_tag not in already_embedded_chunks:
+                            metadatas.append(
+                                {
+                                    "doc_chunk": doc_chunk,
+                                    "link": search_result["link"],
+                                    "chunk_tag": chunk_tag,
+                                    "date_accessed": evidence_fetched_date,
+                                    "query": document["claim"],
+                                }
+                            )
+                            chunks_to_embed += 1
+                        else:
+                            chunks_to_skip += 1
+                    total_chunks += len(doc_chunks)
+
+        encoding = tiktoken.encoding_for_model(self.embedding_model)
+        doc_chunks = [x["doc_chunk"] for x in metadatas]
+        num_words = len(" ".join(doc_chunks).split())
+        num_tokens = len(encoding.encode("".join(doc_chunks)))
+
+        print(
+            f"Created {total_chunks} chunks of text to answer from {len(seen_links)} websites"
+        )
+        print(
+            f"Embedding {chunks_to_embed} (skipping {chunks_to_skip}) chunks of text from {len(seen_links)} websites)"
+        )
+        print(
+            f"Embedding {num_tokens} tokens ({num_words} words) from {len(doc_chunks)} chunks"
+        )
+        print(
+            f"Step5: Estimated cost: {num_tokens * PRICE_PER_1K_TOKENS[self.embedding_model]['embed'] / 1000:.2f} USD"
+        )
+
+        if metadatas:
+            self.compute_embeddings_from_chunks(
+                embedding_function=embedding_function,
+                faiss_db=faiss_db,
+                metadatas=metadatas,
+            )
+
+            return faiss_db
+        return None
+
+    def embed(self):
+        init_openai_with_api_key()
+        src_paths = []
+        for evidence_key in (
+            "google_search_results_evidence",
+            "cohere_wikipedia_evidence",
+        ):
+            evidence_paths = list(PIPELINE_PATHS[evidence_key].glob("**/*.json"))
+            src_paths.extend(evidence_paths)
+
+        if self.filter_str:
+            num_paths = len(src_paths)
+            src_paths = [
+                src_path for src_path in src_paths if self.filter_str in src_path.name
+            ]
+            print(
+                f"Filtering for {self.filter_str} (from {num_paths} to {len(src_paths)})"
+            )
+
+        print(f"Found {len(src_paths)} collections of evidence")
+        src_paths = sorted(src_paths)
+
+        embedding_function = OpenAIEmbeddings()
+        faiss_persist_dir = (
+            PIPELINE_PATHS["faiss_db_embeddings_for_evidence"]
+            / f"{self.embedding_model}_chunk_size_{self.text_embedding_chunk_size}"
+        )
+
+        if faiss_persist_dir.exists():
+            if self.refresh_faiss_db:
+                print(f"Deleting existing database at {faiss_persist_dir}")
+                shutil.rmtree(faiss_persist_dir)
+
+        # check which chunks we've already embedded to avoid duplication
+        if faiss_persist_dir.exists() and not self.refresh_faiss_db:
+            faiss_db = FAISS.load_local(
+                folder_path=str(faiss_persist_dir), embeddings=embedding_function
+            )
+            print(f"Found existing database at {faiss_persist_dir}, using... ")
+        else:
+            index = faiss.IndexFlatL2(EMBEDDING_DIMENSIONS[self.embedding_model])
+            docstore = InMemoryDocstore({})
+            index_to_docstore_id = {}
+            faiss_db = FAISS(
+                embedding_function=embedding_function.embed_query,
+                index=index,
+                docstore=docstore,
+                index_to_docstore_id=index_to_docstore_id,
+            )
+            print(f"Persisting intialised database to {faiss_persist_dir}")
+            faiss_db.save_local(folder_path=str(faiss_persist_dir))
+
+        already_embedded_chunks = {
+            doc.metadata["chunk_tag"] for doc in faiss_db.docstore._dict.values()
+        }
+
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.text_embedding_chunk_size,
+            chunk_overlap=0,
+        )
+
+        kwarg_list = []
+        seen_links = set()
+        metadatas = []
+        total_chunks = 0
+        chunks_to_embed = 0
+        chunks_to_skip = 0
+
+        for src_path in src_paths:
+            with open(src_path, "r") as f:
+                data = json.load(f)
+
+            evidence_fetched_date = self.parse_date_of_fetching(data)
+
+            for document in data["documents"]:
+                for search_result in document["search_results"]:
+                    # Don't embed the same link twice
+                    if search_result["link"] in seen_links:
+                        continue
+                    seen_links.add(search_result["link"])
+
+                    doc_chunks = [
+                        doc.page_content
+                        for doc in splitter.create_documents([search_result["text"]])
+                    ]
+                    chunk_tags = [
+                        f"{search_result['link']}-chunk-{idx}-chunk_sz-{self.text_embedding_chunk_size}"
+                        for idx in range(len(doc_chunks))
+                    ]
+                    for doc_chunk, chunk_tag in zip(doc_chunks, chunk_tags):
+                        if chunk_tag not in already_embedded_chunks:
+                            metadatas.append(
+                                {
+                                    "doc_chunk": doc_chunk,
+                                    "link": search_result["link"],
+                                    "chunk_tag": chunk_tag,
+                                    "date_accessed": evidence_fetched_date,
+                                    "query": document["claim"],
+                                }
+                            )
+                            chunks_to_embed += 1
+                        else:
+                            chunks_to_skip += 1
+                    total_chunks += len(doc_chunks)
+
+        encoding = tiktoken.encoding_for_model(self.embedding_model)
+        doc_chunks = [x["doc_chunk"] for x in metadatas]
+        num_words = len(" ".join(doc_chunks).split())
+        num_tokens = len(encoding.encode("".join(doc_chunks)))
+
+        print(
+            f"Created {total_chunks} chunks of text to answer from {len(seen_links)} websites"
+        )
+        print(
+            f"Embedding {chunks_to_embed} (skipping {chunks_to_skip}) chunks of text from {len(seen_links)} websites)"
+        )
+        print(
+            f"Embedding {num_tokens} tokens ({num_words} words) from {len(doc_chunks)} chunks"
+        )
+        print(
+            f"Estimated cost: {num_tokens * PRICE_PER_1K_TOKENS[self.embedding_model]['embed'] / 1000:.2f} USD"
+        )
+
+        if metadatas:
+            self.compute_embeddings_from_chunks(
+                embedding_function=embedding_function,
+                faiss_persist_dir=faiss_persist_dir,
+                metadatas=metadatas,
+            )

step6_api_claims_to_evidence.py

@@ -0,0 +1,85 @@
+import faiss
+import shutil
+from beartype import beartype
+import numpy as np
+import json
+import argparse
+from zsvision.zs_utils import BlockTimer
+import tiktoken
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import multiprocessing as mp
+from zsvision.zs_multiproc import starmap_with_kwargs
+from llm_api_utils import init_openai_with_api_key, PRICE_PER_1K_TOKENS
+import multiprocessing as mp
+from zsvision.zs_multiproc import starmap_with_kwargs
+from langchain_community.vectorstores import FAISS
+from langchain_openai import OpenAIEmbeddings
+from langchain.docstore.in_memory import InMemoryDocstore
+
+
+class ClaimToEvidence:
+    def __init__(
+        self,
+        embedding_model="ada",
+        limit=0,
+        refresh=False,
+        processes=1,
+        num_chunks_per_worker=50,
+        filter_str="",
+        text_embedding_chunk_size=500,
+        k_nearest_neighbours=3,
+    ):
+        self.embedding_model = embedding_model
+        self.limit = limit
+        self.refresh = refresh
+        self.processes = processes
+        self.num_chunks_per_worker = num_chunks_per_worker
+        self.filter_str = filter_str
+        self.text_embedding_chunk_size = text_embedding_chunk_size
+        self.k_nearest_neighbours = k_nearest_neighbours
+
+    @beartype
+    def link_claims_to_evidence(
+        self,
+        metas,
+        faiss_db,
+    ):
+        embedding_function = OpenAIEmbeddings()
+
+        # build a query from the claim and source fragment
+        queries = [
+            f"Evidence for {x['claim']} (Based on {x['verbatim_quote']})" for x in metas
+        ]
+        encoding = tiktoken.encoding_for_model(self.embedding_model)
+
+        num_tokens = len(encoding.encode(" ".join(queries)))
+        print(
+            f"Step6: Estimated cost: {num_tokens * PRICE_PER_1K_TOKENS[self.embedding_model]['embed'] / 1000:.2f} USD"
+        )
+        k_nearest_neighbours = min(
+            len(faiss_db.index_to_docstore_id), self.k_nearest_neighbours
+        )
+
+        for text_query, meta in zip(queries, metas):
+            docs_and_scores = faiss_db.similarity_search_with_relevance_scores(
+                text_query, k=k_nearest_neighbours
+            )
+
+            # allow evidence to be serialised
+            evidences = []
+            for document, score in docs_and_scores:
+                evidence = {
+                    "chunk_tag": document.metadata["chunk_tag"],
+                    "link": document.metadata["link"],
+                    "query": document.metadata["query"],
+                    "date_accessed": document.metadata["date_accessed"],
+                    "text": document.page_content,
+                    "similarity_score": float(score),
+                }
+                evidences.append(evidence)
+
+            meta["evidences"] = evidences
+            meta["embedded_query_used_to_find_evidence"] = text_query
+
+        print(f"Returning {len(metas)} queries with supporting evidence")
+        return metas

step7_api_check_claims_against_evidence.py

@@ -0,0 +1,153 @@
+import argparse
+import json
+import multiprocessing as mp
+from zsvision.zs_multiproc import starmap_with_kwargs
+from pathlib import Path
+from zsvision.zs_utils import BlockTimer
+from typing import List, Dict
+from llm_api_utils import (
+    call_openai_with_exponetial_backoff,
+    estimate_cost_of_text_generation_api_call,
+    init_openai_with_api_key,
+)
+
+
+class CheckClaimAgainstEvidence:
+    def __init__(
+        self,
+        temperature=0.0,
+        max_num_evidences=2,
+        model="gpt-3.5-turbo",
+        src_dir=Path("data/raw"),
+        dest_dir=Path("data/extracted_claims"),
+        filter_str="",
+        processes=1,
+        refresh=False,
+    ):
+        self.temperature = temperature
+        self.max_num_evidences = max_num_evidences
+        self.model = model
+        self.src_dir = src_dir
+        self.dest_dir = dest_dir
+        self.filter_str = filter_str
+        self.processes = processes
+        self.refresh = refresh
+
+    def check_claim_against_evidence(
+        self,
+        claim: str,
+        evidences: List[Dict[str, str]],
+    ):
+        init_openai_with_api_key()
+        evidence_str = ""
+        for evidence in evidences:
+            # avoid linebreaks in each piece of evidence, else it can create a confusing prompt
+            text_evidence = evidence["text"].replace("\n", " ")
+            evidence_str += f"{text_evidence}\n"
+            evidence_str += f"URL: {evidence['link']}'\n"
+            evidence_str += f"Date accessed: {evidence['date_accessed']}\n\n"
+
+        prompt = f"""\
+Your task is to assess whether a claim is correct based on the given pieces of evidence.
+
+Your answer should be in json format as follows:
+{{
+    "verdict": "<verdict>",
+    "justification": "<justification for the verdict>",
+    "quotes": ["<most relevant verbatim quotes from evidence>"],
+    "URLs": "<URL sources for verbatim quotes>",
+    "date_accessed": "<access dates for URL quotes>"
+}}
+The <verdict> label should be one of the following:
+"Fully supported", "Partially supported", "Unsupported"
+
+When quoting the relevant sentence from the evidence, be careful to copy it **EXACTLY** (with no edits).
+---
+## Example
+
+**Claim**:
+Hannah Arendt was born in 1906.
+
+**Pieces of evidence**:
+Hannah Arendt was a 20th-century German-Jewish political thinker and philosopher. She was born in Linden, Hanover, Germany in 1906. When she was three her family moved to Königsberg so that her father’s syphilis could be treated. He died when she was seven years old. Königsberg was where Immanuel Kant was born, right?
+
+Königsberg was where Immanuel Kant was born, right?
+URL: https://fivebooks.com/best-books/hannah-arendt-samantha-rose-hill/'
+Date accessed: 2023-05-10
+
+Hannah Arendt was born as Johanna Arendt in 1906, in the Wilhelmine period. Her German Jewish family were comfortable, educated and secular in Linden, Prussia (now a part of Hanover). They were merchants of Russian extraction from Königsberg.[a] Her grandparents were members of the Reform Jewish community. Her paternal grandfather, Max Arendt [de] (1843–1913), was a prominent businessman, local politician, a leader of the Königsberg Jewish community and a member of the Centralverein deutscher
+URL: https://en.wikipedia.org/wiki/Hannah_Arendt'
+Date accessed: 2023-05-10
+
+
+**Assessment**:
+{{
+    "verdict": "Fully supported",
+    "justification": "The claim about Hannah Arendt's birth date is directly supported by the evidence."
+    "quote": "Hannah Arendt was born as Johanna Arendt in 1906, in the Wilhelmine period.",
+    "URL": "https://en.wikipedia.org/wiki/Hannah_Arendt",
+    "date_accessed": "2023-05-10"
+}}
+---
+**Claim**:
+{claim}
+
+**Pieces of evidence**:
+{evidence_str}
+**Assessment**:\
+"""
+        persona = "You are a careful research assistant who helps with fact-checking and editing informative articles."
+        system_message = {"role": "system", "content": persona}
+        user_message = {"role": "user", "content": prompt}
+        messages = [system_message, user_message]
+
+        with BlockTimer(
+            f"Using OpenAI API to check claims against evidence {self.model}"
+        ):
+            response = call_openai_with_exponetial_backoff(
+                model=self.model,
+                temperature=self.temperature,
+                messages=messages,
+                response_format={"type": "json_object"},
+            )
+
+        cost = estimate_cost_of_text_generation_api_call(
+            model=self.model, response=response, verbose=True
+        )
+
+        assessment = response.choices[0].message.content
+        assessment_dict = json.loads(assessment)
+        return {"assessment": assessment_dict, "cost": cost}
+
+    def check_claims_against_evidence(self, claims_with_evidence):
+        """
+        Checks claims against evidence.
+        """
+        kwarg_list = []
+        results = []
+        for idx, item in enumerate(claims_with_evidence):
+            kwarg_list.append(
+                {
+                    "claim": item["claim"],
+                    "evidences": item["evidences"][: self.max_num_evidences],
+                }
+            )
+        if self.processes == 1:
+            for kwargs in kwarg_list:
+                results.append(self.check_claim_against_evidence(**kwargs))
+        else:  # multiprocess
+            func = self.check_claim_against_evidence
+            with mp.Pool(processes=self.processes) as pool:
+                results = starmap_with_kwargs(
+                    pool=pool, func=func, kwargs_iter=kwarg_list
+                )
+        costs = [result["cost"] for result in results]
+        print(f"Total cost: {sum(costs)} USD")
+        assessed_claims = []
+        for result, item in zip(results, claims_with_evidence):
+            item["assessment"] = result["assessment"]
+            item["verdict_model"] = self.model
+            assessed_claims.append(item)
+
+        print(f"Writing {len(assessed_claims)} assessed claims")
+        return assessed_claims

step8_api_format_fact_checked_document.py

@@ -0,0 +1,121 @@
+import argparse
+import re
+from collections import defaultdict
+import json
+from text_utils import find_matching_indices
+from pathlib import Path
+
+
+class FormatDocument:
+    def __init__(
+        self,
+        footnote_style: str,
+        temperature=0.0,
+        model="gpt-4",
+        dest_dir=Path("data/extracted_claims"),
+        filter_str="",
+        refresh=False,
+    ):
+        self.temperature = temperature
+        self.model = model
+        self.dest_dir = dest_dir
+        self.filter_str = filter_str
+        self.refresh = refresh
+        self.footnote_style = footnote_style
+
+    def cleanup_explanation(self, claim_assessment: dict, mode: str) -> str:
+        claim = claim_assessment["claim"]
+        assessment = claim_assessment["assessment"]
+        justification = assessment["justification"]
+        category = assessment["verdict"]
+        urls = assessment["URLs"]
+        date_accessed = assessment["date_accessed"]
+
+        prefixes = {
+            "Fully supported": "✅",
+            "Partially supported": "❓",
+            "Unsupported": "❗",
+        }
+        prefix = prefixes[category]
+        quotes = ",".join(f'"{quote}"' for quote in assessment["quotes"])
+        # Sometimes, the verdict justification contains newlines , which messes up the formatting of footnotes.
+        justification = justification.replace("\n", "")
+
+        if mode == "terse":
+            footnote = f"Claim: {claim} 👉 {category} {urls}"
+        elif mode == "verbose":
+            footnote = f"Claim: {claim} 👉 {category} {quotes} {justification}, URLs: {urls}, date accessed: {date_accessed}"
+        footnote = f"{prefix} {footnote}"
+        return footnote
+
+    def reformat_document_to_include_claims(
+        self,
+        original_text,
+        fact_verdicts,
+        footnote_style=None,
+    ):
+        bibliography = []
+        footnote_markers_to_insert = []
+        statistics = defaultdict(int)
+        number_of_facts_checked = 0
+        if footnote_style:
+            self.footnote_style = footnote_style
+        for fact_idx, claim_assessment in enumerate(fact_verdicts):
+            if self.footnote_style == "terse":
+                footnote_str = f"{fact_idx + 1}"
+            elif self.footnote_style == "verbose":
+                footnote_str = claim_assessment["claim"].replace(" ", "-")
+                # footnote markers cannot contain much punctuation or commas in Jekyll
+                # (even though this is valid in GitHub-flavoured markdown)
+                for char in [
+                    ",",
+                    ".",
+                    '"',
+                    "'",
+                    ":",
+                    ";",
+                    "(",
+                    ")",
+                    "[",
+                    "]",
+                    "{",
+                    "}",
+                    "*",
+                ]:
+                    footnote_str = footnote_str.replace(char, "")
+
+            explanation = self.cleanup_explanation(
+                claim_assessment, mode=self.footnote_style
+            )
+            footnote_marker = f"[^{footnote_str}]"
+            query = claim_assessment["verbatim_quote"]
+
+            assert (
+                original_text.count(query) == 1
+            ), f"Found {original_text.count(query)} matches for {query}, rather than 1"
+            start_pos = original_text.find(query)
+            assert start_pos != -1, f"Could not find {query} in {original_text}"
+            end_pos = start_pos + len(query)
+            footnote_markers_to_insert.append((end_pos, footnote_marker))
+            verdict_category = claim_assessment["assessment"]["verdict"]
+            statistics[verdict_category] += 1
+            number_of_facts_checked += 1
+            bibliography.append(f"{footnote_marker}: {explanation} ")
+
+        # perform insertions in reverse order so that the indices don't get messed up
+        modified_text = original_text
+        for char_pos, footnote_marker in sorted(
+            footnote_markers_to_insert, reverse=True
+        ):
+            modified_text = (
+                modified_text[:char_pos] + footnote_marker + modified_text[char_pos:]
+            )
+
+        modified_text += "\n\n"
+        modified_text += "\n".join(bibliography)
+
+        # assert number_of_facts_checked != 0, "No facts were checked"
+        if number_of_facts_checked == 0:
+            print("No objective facts were found.")
+            modified_text = "No clear-cut objective claims were detected."
+        return modified_text

subjective_claims.txt

@@ -0,0 +1 @@
+Arendt's experiences during this time influenced her work on totalitarianism and human rights.

text_utils.py

@@ -0,0 +1,188 @@
+import re
+from typing import Dict
+import unittest
+
+
+def parse_passage_quote_and_claim(passage_quote_and_claim: str) -> Dict[str, str]:
+    """Parse the quote and claim from a string, where the string is of the form:
+
+    [VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] <passage quote for claim> [CLAIM] <claim>
+    """
+
+    if not passage_quote_and_claim.startswith("[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM]"):
+        raise ValueError(f"Invalid input format: {passage_quote_and_claim}")
+
+    parts = passage_quote_and_claim.split("[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM]")
+    source_parts = parts[1].split("[CLAIM]")
+
+    # If there aren't exactly two parts after splitting by [CLAIM], the format is invalid
+    if len(source_parts) != 2:
+        raise ValueError(f"Invalid input format: {passage_quote_and_claim}")
+
+    passage_quote_for_claim = source_parts[0].strip()
+    claim = source_parts[1].strip()
+    return {"verbatim_quote": passage_quote_for_claim, "claim": claim}
+
+
+def is_unique_verbatim_quote(verbatim_quote: str, original_passage: str):
+    """Check if the verbatim quote is an exact quote from the original passage."""
+    return original_passage.count(verbatim_quote) == 1
+
+
+def find_matching_indices(query: str, original_text: str):
+    # Function to remove markdown links and create an index map
+    def remove_links(text):
+        index_map = []
+        result = []
+        markdown_links = re.finditer(r"\[([^\]]+)\]\([^)]+\)", text)
+
+        prev_end = 0
+        for match in markdown_links:
+            result.append(text[prev_end : match.start()])
+            index_map.extend(range(prev_end, match.start()))
+            result.append(match.group(1))
+            index_map.extend(range(match.start(1), match.end(1)))
+            prev_end = match.end()
+
+        result.append(text[prev_end:])
+        index_map.extend(range(prev_end, len(text)))
+
+        return "".join(result), index_map
+
+    # Remove markdown links from the original text and create an index map
+    cleaned_text, index_map = remove_links(original_text)
+
+    # Remove markdown links from the query
+    cleaned_query, _ = remove_links(query)
+
+    # Find the start index of the cleaned query in the cleaned text
+    start = cleaned_text.find(cleaned_query)
+
+    # If the query is not found, return an empty list
+    if start == -1:
+        return []
+
+    # Add the query length to get the end index
+    end = start + len(cleaned_query)
+
+    # Use the index map to find the corresponding start and end indices in the original text
+    original_start = index_map[start]
+    original_end = index_map[end - 1] + 1
+
+    return [(original_start, original_end)]
+
+
+class TestCases(unittest.TestCase):
+    def test_find_matching_indices(self):
+        """Test the find_matching_indices() function.
+        This function should return a list of matches, where each match is a tuple of (start, end) indices.
+
+        The start and end indices should be the character positions of the query in the original_text, accounting
+        for the fact that markdown links should be ignored when performing the match.
+
+        """
+        test_cases = [
+            {
+                "query": "Her writings on these subjects continue to resonate with present-day political struggles, as well as with the works of other philosophers like Immanuel Kant and Edmund Husserl.",
+                "original": "Arendt's later works, sought to further unravel the complexities of power and rebellion. Her writings on these subjects continue to resonate with present-day political struggles, as well as with the works of other philosophers like [Immanuel Kant](/philosophy/2023-immanuel-kant-life-and-work) and [Edmund Husserl](/philosophy/2023-edmund-husserl-his-life-and-work).\n\n## A Lasting Legacy",
+                "expected": "Her writings on these subjects continue to resonate with present-day political struggles, as well as with the works of other philosophers like [Immanuel Kant](/philosophy/2023-immanuel-kant-life-and-work) and [Edmund Husserl](/philosophy/2023-edmund-husserl-his-life-and-work).",
+            },
+            {
+                "query": "I went to the sea side (at the weekend).",
+                "original": "I woke up. Then I went to the sea side (at the weekend). Then I went home.",
+                "expected": "I went to the sea side (at the weekend).",
+            },
+            {
+                "query": "no merger with the [solar farm] company",
+                "original": "There would be no merger with the [solar farm] company.",
+                "expected": "no merger with the [solar farm] company",
+            },
+            {
+                "query": "with the works of other philosophers like [Immanuel Kant](/philosophy/2023-immanuel-kant-life-and-work) and [Edmund Husserl](/philosophy/2023-edmund-husserl-his-life-and-work)",
+                "original": "\n\n## Fleeing Germany and the Road to Academia\n\nWith the rise of the Nazi regime in the 1930s, Arendt's Jewish heritage put her in grave danger. She fled Germany in 1933 and settled in Paris, where she became involved with a number of political and social organizations advocating for Jewish refugees. In 1940, she was interned in a French camp as an enemy alien, but managed to escape and eventually make her way to the United States in 1941.\n\nArendt's experiences during this time would deeply influence her work on totalitarianism and human rights. In New York, she began to immerse herself in academic life, working as an editor, journalist, and lecturer. Her first major work, *The Origins of Totalitarianism*, published in 1951, explored the common roots of Nazism and Stalinism, and established her as a significant voice in political philosophy.\n\n## A Life Of Controversial, Influential Works\n\nThroughout her career, Arendt wrote a number of seminal, and controversial, works. *The Human Condition* (1958) examined the role of politics in modern societies and introduced the concept of \"the public realm\" – the space where individuals act and participate in political life. This exploration of freedom and action would become a recurring theme in her writings.\n\nHer 1963 publication, *Eichmann in Jerusalem: A Report on the Banality of Evil*, based on her coverage of Adolf Eichmann's trial, ignited significant controversy. Arendt argued that Eichmann, a key architect of the Holocaust, was not a monster but rather an ordinary bureaucrat who unquestioningly followed orders. The idea of the \"banality of evil\" continues to influence discussions on the nature of evil and moral responsibility.\n\nArendt's later works, such as *On Revolution* (1963) and *Between Past and Future* (1968), sought to further unravel the complexities of power, authority, and rebellion. Her writings on these subjects continue to resonate with present-day political struggles, as well as with the works of other philosophers like [Immanuel Kant](/philosophy/2023-immanuel-kant-life-and-work) and [Edmund Husserl](/philosophy/2023-edmund-husserl-his-life-and-work).\n\n## A Lasting Legacy\n\nHannah Arendt died in 1975, but her work remains as relevant as ever.",
+                "expected": "with the works of other philosophers like [Immanuel Kant](/philosophy/2023-immanuel-kant-life-and-work) and [Edmund Husserl](/philosophy/2023-edmund-husserl-his-life-and-work)",
+            },
+        ]
+
+        for test_case in test_cases:
+            matches = find_matching_indices(
+                query=test_case["query"], original_text=test_case["original"]
+            )
+            assert (
+                len(matches) == 1
+            ), f"Expected exactly one match, but found {len(matches)}"
+            result = test_case["original"][matches[0][0] : matches[0][1]]
+            msg = (
+                f"Expected\n\n{test_case['expected']}\n\nbut instead found\n\n{result}"
+            )
+            self.assertEqual(result, test_case["expected"], msg)
+        print(f"Passed all tests for find_matching_indices()")
+
+    def test_parse_passage_quote_and_claim(self):
+        """Test the following function:
+            parse_passage_quote_and_claim(passage_quote_and_claim: str) -> {"verbatim_quote": str, "claim": str}
+
+        The passage quote and claim should take the form:
+        [VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] <passage quote for claim> [CLAIM] <claim>
+        """
+        test_cases = [
+            {
+                "passage_quote_and_claim": "[VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] Hannah Arendt [was born in] 1906 in Linden, Germany [CLAIM] Hannah Arendt was born in Linden, Germany.",
+                "expected": {
+                    "verbatim_quote": "Hannah Arendt [was born in] 1906 in Linden, Germany",
+                    "claim": "Hannah Arendt was born in Linden, Germany.",
+                },
+            },
+            {
+                "passage_quote_and_claim": "Something [VERBATIM_PASSAGE_QUOTE_FOR_CLAIM] Hannah Arendt [was born in] 1906 in Linden, Germany [CLAIM] Hannah Arendt was born in Linden, Germany.",
+                "expected": "Exception",
+            },
+        ]
+        for test_case in test_cases:
+            expected = test_case["expected"]
+            if expected == "Exception":
+                self.assertRaises(
+                    ValueError,
+                    parse_passage_quote_and_claim,
+                    test_case["passage_quote_and_claim"],
+                )
+            else:
+                parsed = parse_passage_quote_and_claim(
+                    passage_quote_and_claim=test_case["passage_quote_and_claim"]
+                )
+                self.assertEqual(parsed["verbatim_quote"], expected["verbatim_quote"])
+
+    def test_is_unique_verbatim_quote_check(self):
+        """Test the following function:
+            is_unique_verbatim_quote_check(verbatim_quote: str) -> bool
+
+        This function should return True if the verbatim quote is indeed a quote and is unique, and false otherwise.
+
+        """
+        test_cases = [
+            {
+                "verbatim_quote": "Hannah Arendt [was born in] 1906 in Linden, Germany",
+                "original_passage": "Hannah Arendt [was born in] 1906 in Linden, Germany at a time when...",
+                "expected": True,
+            },
+            {
+                "verbatim_quote": "Hannah Arendt [was born in] 1906 in Linden, Germany",
+                "original_passage": "Hannah Arendt [wasn't born in] 1906 in Linden, Germany at a time when...",
+                "expected": False,
+            },
+            {
+                "verbatim_quote": "Hannah Arendt [was born in] 1906 in Linden, Germany. Hannah Arendt was a person.",
+                "original_passage": "Hannah Arendt",
+                "expected": False,
+            },
+        ]
+        for test_case in test_cases:
+            result = is_unique_verbatim_quote(
+                verbatim_quote=test_case["verbatim_quote"],
+                original_passage=test_case["original_passage"],
+            )
+            self.assertEqual(result, test_case["expected"])
+
+
+if __name__ == "__main__":
+    unittest.main()

utils.py

@@ -0,0 +1,18 @@
+import os
+from functools import lru_cache
+from langchain_community.utilities import GoogleSearchAPIWrapper
+
+@lru_cache(maxsize=2)
+def get_search_wrapper():
+    os.environ["GOOGLE_API_KEY"] = os.environ.get("GOOGLE_CLOUD_API_KEY")
+
+    os.environ["GOOGLE_CSE_ID"] = os.environ.get("GOOGLE_CUSTOM_SEARCH_ENGINE_ID")
+    return GoogleSearchAPIWrapper()
+
+
+def get_google_search_results(query_str: str, num_results: int):
+    google_search_tool = get_search_wrapper()
+    search_results = google_search_tool.results(
+        query=query_str, num_results=num_results
+    )
+    return search_results