init

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+.gradio/
+temp/*
+.cache/*
+nohup.out

README.md

@@ -0,0 +1,12 @@
+---
+title: Fathom DeepResearch
+emoji: 📊
+colorFrom: green
+colorTo: red
+sdk: gradio
+sdk_version: 5.44.1
+app_file: app.py
+pinned: false
+license: mit
+short_description: Use the fathom search 4b model interactively
+---

eval_benchmarks.py

@@ -0,0 +1,468 @@
+# eval_benchmark_multithreaded.py
+"""Unified benchmarking script for ReCall, ZeroSearch, and R1‑Searcher
+with optional multi‑threaded execution.
+
+Example usage (single‑threaded)
+-------------------------------
+```bash
+python eval_benchmark.py \
+    --dataset frames \
+    --agent r1-searcher \
+    --model-url http://0.0.0.0:1233 \
+    --out-base /tmp/evals \
+    --mode single
+```
+
+Example usage (multi‑threaded, 128 workers)
+------------------------------------------
+```bash
+python eval_benchmark.py \
+    --dataset frames \
+    --agent recall \
+    --model-url http://0.0.0.0:1231 \
+    --out-base /tmp/evals \
+    --mode multi \
+    --workers 128
+```
+The script will:
+1. Load the specified dataset JSONL file that contains objects with keys
+   `question` and `answer`.
+2. Build the chosen agent wrapper (`recall`, `zerosearch`, or `r1-searcher`).
+3. Stream one JSONL line per example with *all* details needed for analysis.
+4. Optionally run the evaluation loop in parallel using a configurable number
+   of worker threads.
+5. Automatically construct the output path as:
+   ```
+   {out_base}/{model_name}/{dataset}.jsonl
+   ```
+   where `model_name` is derived from the `--model-url` (characters after the
+   last `/`).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import pathlib
+import re
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Dict, List
+
+import unicodedata
+from openai import OpenAI, APIStatusError
+from tqdm import tqdm
+
+# --------------------------------------------------------------------
+# Agent imports (ensure PYTHONPATH is set appropriately)
+# --------------------------------------------------------------------
+from re_call import ReCall  # user's wrapper
+# from re_call import ZeroSearchInference, ZeroSearchConfig
+# from re_call import R1Searcher, R1SearchConfig as R1Cfg
+# from re_call import O1Cfg, O1Searcher
+from pathlib import Path
+# from re_call import SDSCfg, SDSSearcher
+
+# --------------------------------------------------------------------
+# Environment Keys – override with real keys or environment variables
+# --------------------------------------------------------------------
+#for recall
+# search_env = "from search_api import web_search, web_visit"
+# search_schemas =[
+#                                         {
+#                                             "name": "web_search",
+#                                             "description": "Google search and return links to web-pages with a brief snippet given a text query",
+#                                             "parameters": {
+#                                                 "type": "object",
+#                                                 "properties": {
+#                                                     "query": {"type": "string"},
+#                                                 },
+#                                                 "required": ["query"],
+#                                             },
+#                                         },
+#                                         {
+#                                             "name": "web_visit",
+#                                             "description": "Visit webpage and return its content",
+#                                             "parameters": {
+#                                                 "type": "object",
+#                                                 "properties": {
+#                                                     "url": {"type": "string",  "description": "The URL of the webpage to visit. Must be a single URL"},
+#                                                 },
+#                                                 "required": ["url"],
+#                                             },
+#                                         }
+#                 ]
+# for recall
+search_env = "from search_api import search_urls, open_url, search_and_parse_query, query_url"
+search_schemas =[
+                                        {
+                                            "name": "search_urls",
+                                            "description": "Google search and return links to web-pages with a brief snippet given a text query",
+                                            "parameters": {
+                                                "type": "object",
+                                                "properties": {
+                                                    "query": {"type": "string"},
+                                                    "top_k": {"type": "integer", "default": 10},
+                                                },
+                                                "required": ["query"],
+                                            },
+                                        },
+                                        {
+                                            "name": "query_url",
+                                            "description": "Visit webpage and return evidence based retrival for the provided goal",
+                                            "parameters": {
+                                                "type": "object",
+                                                "properties": {
+                                                    "url": {"type": "string",  "description": "The URL of the webpage to visit. Must be a single URL"},
+                                                    "goal": {"type": "string",  "description": "The specific information goal for visiting webpage"},
+                                                },
+                                                "required": ["url", "goal"],
+                                            },
+                                        }
+                ]
+
+EXECUTOR_URL = os.environ["HOST_SERPER_URL"]
+DATA_ROOT = pathlib.Path("./eval_datasets")
+SEM = threading.Semaphore(3)  # limit concurrent judge calls
+JUDGE_MODEL = "gpt-4.1-mini"
+
+try:
+    base = Path(__file__).resolve().parent
+except NameError:  # e.g., REPL/Jupyter
+    base = Path.cwd()
+    
+TOKENIZER_DIR = (base / "tokenizer-info").resolve()
+
+# ───────────────────────── tokenizer ────────────────────────────────────────
+try:
+    from transformers import AutoTokenizer
+    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
+except Exception as e:
+    import sys 
+    sys.exit(f"❌  Could not load Qwen3 tokenizer: {e}")
+
+import hashlib
+
+def get_uid(sample: dict) -> str:
+    """Generate a UID using SHA256 hash of question."""
+    return hashlib.sha256(sample["question"].strip().encode("utf-8")).hexdigest()
+
+# --------------------------------------------------------------------
+# Regex & utilities
+# --------------------------------------------------------------------
+def extract_answer_tagged(text: str) -> str:
+
+    ANS_RE = re.compile(r"<answer>(.*?)</answer>", re.S)
+    match = ANS_RE.findall(text)
+    if match :
+        return match[-1].strip().lower() 
+    else:
+        print("No answer tags found")
+        return text[-200:] #because o1-searcher fails to follow format 
+
+def extract_answer_boxed(response):
+    def remove_boxed(s):
+        if "\\boxed " in s:
+            left = "\\boxed "
+            assert s[:len(left)] == left
+            return s[len(left):]
+
+        left = "\\boxed{"
+
+        assert s[:len(left)] == left
+        assert s[-1] == "}"
+
+        return s[len(left):-1]
+
+    def last_boxed_only_string(string):
+        idx = string.rfind("\\boxed")
+        if "\\boxed " in string:
+            return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0]
+        if idx < 0:
+            idx = string.rfind("\\fbox")
+            if idx < 0:
+                return None
+
+        i = idx
+        right_brace_idx = None
+        num_left_braces_open = 0
+        while i < len(string):
+            if string[i] == "{":
+                num_left_braces_open += 1
+            if string[i] == "}":
+                num_left_braces_open -= 1
+                if num_left_braces_open == 0:
+                    right_brace_idx = i
+                    break
+            i += 1
+
+        if right_brace_idx is None:
+            retval = None
+        else:
+            retval = string[idx:right_brace_idx + 1]
+
+        return retval
+    answer = remove_boxed(last_boxed_only_string(response))
+    return answer
+
+
+
+JUDGE_SYS = """
+You are an impartial judge evaluating the correctness of a model's answer against a ground-truth answer for a given question. Your task is to:
+1. Compare the model's answer to the ground-truth answer.
+2. Determine if the model's answer is correct or incorrect.
+
+**Input Format:**
+- Question: {question}
+- Ground Truth: {ground_truth}
+- Model Answer: {model_answer}
+
+**Output Format:**
+ correct/incorrect/unknown
+
+**Guidelines:**
+- The model's answer is correct if it matches the ground-truth answer in meaning and content it is case-insensitive, ignore minor punctuation or formatting differences.
+- If the model's answer contains additional information, it is still correct as long as the core answer matches the ground truth.
+- Be precise output a single word correct / incorrect / unknown and **nothing else**
+- For MCQ questions match the option ID A. B. C. or D. if its correct the answer is correct. 
+"""
+# - If the model's answer is partially correct or contains errors, it is incorrect.
+
+
+# Thread‑local OpenAI client cache
+
+def _oa() -> OpenAI:
+    th = threading.current_thread()
+    if not hasattr(th, "_oa"):
+        th._oa = OpenAI()
+    return th._oa
+
+
+def judge(q: str, gt: str, pred: str) -> str:
+    if pred == "":
+        return "unknown"
+    prompt = JUDGE_SYS.format(question=q, ground_truth=gt, model_answer=pred)
+    try:
+        with SEM:
+            resp = _oa().chat.completions.create(
+                model=JUDGE_MODEL,
+                messages=[
+                    {"role": "system", "content": JUDGE_SYS},
+                    {"role": "user", "content": prompt},
+                ],
+                temperature=0.0,
+                max_tokens=100,
+            )
+        return resp.choices[0].message.content.strip().lower()
+    except APIStatusError:
+        return "unknown"
+
+
+# --------------------------------------------------------------------
+# Agent factory
+# --------------------------------------------------------------------
+def build_agent(kind: str, model_url: str):
+    kind = kind.lower()
+    print(kind)
+    if kind == "recall":
+        return ReCall(executor_url=EXECUTOR_URL)
+    else:
+        raise ValueError(f"Unknown agent kind: {kind}")
+    # if kind == "o1-search" or kind == "sds":
+    #     cfg = O1Cfg()
+    #     return O1Searcher(cfg, thinker_url=model_url)
+    # if kind == "zerosearch":
+    #     cfg = ZeroSearchConfig(thinker_url=model_url)
+    #     return ZeroSearchInference(cfg)
+    # if kind in ("r1-search", "r1-searcher", "r1"):
+    #     cfg = R1Cfg(serper_api_key=os.getenv("SERPER_API_KEY", ""))
+    #     return R1Searcher(cfg=cfg, model_url=model_url)
+    # raise ValueError(f"Unknown agent kind: {kind}")
+
+
+# --------------------------------------------------------------------
+# Core evaluation routine for a single example (thread‑safe)
+# --------------------------------------------------------------------
+def evaluate_example(example: Dict[str, str], agent_kind: str, model_url: str) -> Dict[str, str]:
+    """Run one example through the pipeline and return result row."""
+    question = example["question"].strip()
+    answer_gt = example["answer"].strip()
+    idx = example["id"].strip()
+
+
+    # Build a *fresh* agent per thread to avoid shared‑state issues
+    agent = build_agent(agent_kind, model_url=model_url)
+
+    if agent_kind ==  "recall" and model_url == "deepseek-ai/DeepSeek-R1":
+        # print(agent_kind)
+        # print("B"*100)
+        transcript, tool_calls  = agent.run_deepseek(
+            env=search_env,
+            func_schemas=search_schemas,
+            question=question,
+            model_name="deepseek-ai/DeepSeek-R1",
+            temperature=0.6,
+            max_tokens=40960, 
+            # tokenizer = tokenizer
+        )
+    elif agent_kind == "recall":
+        transcript, tool_calls, chat = agent.run(
+            env=search_env,
+            func_schemas=search_schemas,
+            question=question,
+            model_url=model_url,
+            temperature=0.6,
+            max_new_tokens=40960, 
+            tokenizer = tokenizer
+        )
+        # tool_calls = agent.extract_tool_calls(transcript)
+    else:  # zerosearch or r1‑searcher
+        transcript, tool_calls = agent.run(question)
+
+    if agent_kind in [
+        "r1-searcher",
+        "zerosearch",
+        # "o1-search",
+    ]:
+        pred = extract_answer_tagged(transcript)
+    if agent_kind in [
+        "recall",
+        "SDS"
+        "o1-searcher"
+    ]:  
+        try:
+            pred = extract_answer_boxed(transcript)
+        except:
+            print("falling to last string")
+            pred = transcript[-200:]
+    else:
+        try:
+            pred = extract_answer_boxed(transcript)
+        except:
+            print("falling to last string")
+            pred = transcript[-200:]
+
+    verdict = judge(question, answer_gt.lower(), pred.lower())
+
+    return {
+        "id": idx,
+        "question": question,
+        "answer_gt": answer_gt,
+        "model_answer": pred,
+        "judge": verdict,
+        "tool_calls": tool_calls,
+        "transcript": transcript,
+        "chat": chat
+    }
+
+# --------------------------------------------------------------------
+# CLI entry‑point
+# --------------------------------------------------------------------
+def build_output_path(out_base, agent, dataset, name) -> pathlib.Path:
+    """Construct output path as {out_base}/{model_name}/{dataset}.jsonl."""
+    return out_base  / f"{agent}" / f"{dataset}-{name}.jsonl"
+
+def normalize(s: str) -> str:
+    return unicodedata.normalize("NFKD", s.strip().lower())
+
+def load_existing_results(path: pathlib.Path) -> tuple[list[dict], set[str]]:
+    results = []
+    uids = set()
+    if not path.exists():
+        return results, uids
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            try:
+                row = json.loads(line)
+                if row['model_answer'] != "":
+                    results.append(row)
+                    uids.add(row["id"])
+            except Exception:
+                continue
+    return results, uids
+
+def main():
+    parser = argparse.ArgumentParser(description="Benchmark QA agents on a dataset (single or multi‑threaded)")
+    parser.add_argument("--dataset", required=True, help="dataset name (frames, …)")
+    parser.add_argument("--agent", required=True, choices=["recall", "zerosearch", "r1-searcher", "o1-search", "SDS", "deepseek-r1"], help="agent wrapper")
+    parser.add_argument("--out", required=True, help="base directory for outputs")
+    parser.add_argument("--model-url", required=False, help="URL of the model server")
+    parser.add_argument("--limit", type=int, default=0, help="optional cap on number of questions")
+    parser.add_argument("--mode", choices=["single", "multi"], default="single", help="execution mode")
+    parser.add_argument("--workers", type=int, default=8, help="number of worker threads for multi‑mode")
+    parser.add_argument("--name", type=str, default="", help="suffix for save dir")
+
+    args = parser.parse_args()
+
+    # ----------------------------------------------------------------
+    # Dataset loading
+    # ----------------------------------------------------------------
+    ds_path = DATA_ROOT / f"{args.dataset}.jsonl"
+    if not ds_path.exists():
+        raise FileNotFoundError(ds_path)
+
+    with ds_path.open() as f:
+        data = [json.loads(line) for line in f]
+
+    # ----------------------------------------------------------------
+    # Output path setup
+    # ----------------------------------------------------------------
+    out_base = pathlib.Path(args.out).expanduser().resolve()
+    out_path = build_output_path(out_base, args.agent, args.dataset, args.name)
+    print(out_path)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if args.limit:
+        data = data[: args.limit]
+    # data = data[246:]
+
+    correct = 0
+    start_time = time.perf_counter()
+
+
+    # ----------------------------------------------------------------
+    # SINGLE‑THREADED EXECUTION
+    # ----------------------------------------------------------------
+    if args.mode == "single":
+        with open(out_path, "w", encoding="utf-8") as fout:
+            for ex in tqdm(data, desc="QA loop (single)"):
+
+                row = evaluate_example(ex, args.agent, args.model_url)
+                if row["judge"] == "correct":
+                    correct += 1
+                # context for row
+                row.update({"agent": args.agent, "dataset": args.dataset})
+                fout.write(json.dumps(row, ensure_ascii=False) + "\n")
+                fout.flush()
+
+    # ----------------------------------------------------------------
+    # MULTI‑THREADED EXECUTION
+    # ----------------------------------------------------------------
+    else:
+        workers = max(1, args.workers)
+        logging.info("Running in multi‑threaded mode with %d workers", workers)
+        with ThreadPoolExecutor(max_workers=workers) as executor, open(out_path, "a", encoding="utf-8") as fout:
+            futures = {executor.submit(evaluate_example, ex, args.agent, args.model_url): ex for ex in data}
+            for fut in tqdm(as_completed(futures), total=len(futures), desc="QA loop (multi)"):
+                try:
+                    row = fut.result()
+                except Exception as exc:
+                    logging.exception("Evaluation failed: %s", exc)
+                    continue
+                # print(row['id'])
+                if row["judge"] == "correct":
+                    correct += 1
+                row.update({"agent": args.agent, "dataset": args.dataset})
+                fout.write(json.dumps(row, ensure_ascii=False) + "\n")
+                fout.flush()
+
+    elapsed = time.perf_counter() - start_time
+    accuracy = correct / len(data) if data else 0.0
+    print(f"Accuracy: {correct}/{len(data)} = {accuracy:.1%}")
+    print(f"Elapsed time: {elapsed:.2f}s ({elapsed/len(data):.2f}s per example)")
+
+
+if __name__ == "__main__":
+    main()

re_call/__init__.py

@@ -0,0 +1,8 @@
+# from .inference.re_call import ReCall
+# from .inference.r1_searcher import R1Searcher, R1SearchConfig
+# from .inference.zerosearch import ZeroSearchInference, ZeroSearchConfig
+# from .inference.o1_searcher import O1Cfg, O1Searcher
+# from .inference.simpledeepsearch import SDSCfg, SDSearcher
+from .re_call import ReCall
+__all__ = ["ReCall"]
+# __all__ = ["ReCall", "R1Searcher", "ZeroSearchInference", "ZeroSearchConfig", "R1SearchConfig", "O1Cfg", "O1Searcher", "SDSCfg", "SDSearcher"]

re_call/prompts.py

@@ -0,0 +1,184 @@
+from typing import Final
+ 
+# DEEPRESEARCH_REPORT_SYS_PROMPT: Final[str] = r"""
+# You are a DeepResearch analyst and Report Converter. Turn a raw investigation trace into a clear,
+# decision-grade report suitable for executives.
+ 
+# INPUTS (provided in the user message)
+# - QUESTION: the research question.
+# - TRACE: the full transcript (may include assistant/user/tool snippets).
+# - TOOL_CALLS: raw list of tool calls (JSON-ish), which may contain URLs.
+ 
+# CRITICAL SOURCING CONSTRAINTS (non-negotiable)
+# - TRAJECTORY_LINKS = every URL you find in TRACE and TOOL_CALLS. Use ONLY these links. Do NOT add new sources.
+# - Evidence density: cite every non-obvious fact/date/figure/evaluative claim.
+# - Citation format: append raw bracketed URLs immediately after the supported sentence/point,
+#   e.g., “… announced in 2003. [https://example.com/page]”.
+# - Prefer primary/official and the most recent authoritative updates. If sources conflict, explain briefly and cite both.
+ 
+# QUALITY & FRESHNESS
+# - Be neutral, precise, and reproducible. No fabrication.
+# - Distinguish **event date**, **publish/update date**, and **effective date** where relevant.
+# - If critical info is missing, state the gap and proceed with best-effort analysis grounded in available links.
+ 
+# OUTPUT RULES
+# - **Markdown only.** No system markers. No boxed answers (\boxed{}).
+# - Public-facing rationale only (no hidden chain-of-thought).
+# - Length proportional to complexity (short for simple, detailed for complex).
+# - **You decide the sectioning and narrative flow** based on the QUESTION and TRACE. Use headings only if they help clarity.
+# - Keep it decision-useful: tight claims tied to evidence, crisp takeaways, explicit uncertainties.
+ 
+# OPERATION
+# 1) Extract TRAJECTORY_LINKS from TRACE and TOOL_CALLS. These are your only allowable citations.
+# 2) Think privately about the best structure for this topic; then write the report accordingly.
+# 3) Map each included claim to at least one link; mark any necessary but unsupported claim as “unsupported”.
+# 4) Normalize names/dates/figures; note gaps and conflicts, and how you resolved them.
+# 5) Conclude with a **deduplicated “Sources used” list** of the raw URLs you actually cited (one per line).
+ 
+# """
+
+DEEPRESEARCH_SYS_PROMPT: Final[str] = r"""
+You are a DeepResearch Assistant.
+
+Goal: (1) Produce a concise PLAN that breaks the QUESTION into sections and **maps every URL and tool_call content** in the trace to those sections; (2) Produce a public-facing REPORT that synthesizes **all** information from TRACE/TOOL_CALLS into an insightful report.
+
+========================
+INPUTS
+========================
+- QUESTION: research question.
+- TRACE: transcript (assistant/user/tool snippets).
+- TOOL_CALLS: raw tool calls (includes URLs and tool_responses).
+
+
+========================
+CITATIONS (ACCURACY-FIRST)
+========================
+- **TRAJECTORY_LINKS** = all URLs in TRACE/TOOL_CALLS. Cite **only** these; do not invent/browse.
+-  Cite pivotal or non-obvious claims (dates, numbers, quotes, contested points).
+- **Density with accuracy:** Prefer **dense citations** on non-obvious/pivotal claims **only when confident** the link supports the exact statement; avoid stray/low-confidence citations.
+- **Sources used** = only URLs actually cited in REPORT.
+-  Citation format: append raw square bracketed full URLs immediately after the supported sentence/point, e.g., “… announced in 2003. [https://example.com/page]”.
+
+
+========================
+PLAN (MANDATORY CONTENT)
+========================
+1) **Question → Sections** (derivation):
+   - Decompose QUESTION into sub-questions SQ1..SQn, then plan the structure of the report around that to cover all bases.
+   - Clearly outline the breakdown and structure of the report and the thought process for it.
+
+2) **Evidence Map: Section → URL/tool_call mapping**
+   - **Harvest** all URLs from TRACE and TOOL_CALLS → this forms TRAJECTORY_LINKS.
+   - For **each Section (S1..Sn)**, list the **evidence items** (every TRAJECTORY_LINK and its content explored in the TRACE) relevant to it.
+   - **Coverage rule:** Ensure **most** URL/tool_call items from TRACE is mapped to at least one Section (unless truly irrelevant to the topic).
+   - Use this table (include all rows; add as many as needed):
+   | Section | Item | | Content | Confidence |
+   |---|---|---|---|---|
+   | S1 | <URL_4>  | date/stat/quote/context | High/Med/Low |
+   | S2 | <URL_1>  <URL_2>  | stat/definition/quote | High/Med/Low |
+   - If something is truly irrelevant, list under **Omitted as Irrelevant (with reason)**; keep this list short do not cite them in the report in this case.
+
+3) **Layout the Strategy for insight generation**:
+   - 4–6 bullets on how you will generate higher level insight / aalysis: e.g., contrast/benchmark, timeline, ratios/growth, causal chain, risks.
+   - You may generate insights / analysis by concatenating **general background knowledge** with TRACE facts, but only if the TRACE facts remain central.
+   - Beyond description, provide **analysis, interpretation, and recommendations** where possible.
+   - Recommendations must be **derived strictly from TRACE evidence**. No hallucinated numbers or unsupported claims.
+   - If evidence is insufficient for a clear recommendation, state this explicitly.
+
+========================
+REPORT (MANDATORY CONTENT)
+========================
+- # Executive Summary — 5-10 crisp bullets with concrete takeaways; cite pivotal/non-obvious claims.
+- ## Main Body — brief scope and inclusion rules; **provide higher-order insights built on the harvested evidence** (e.g., causal explanations, benchmarks, ratios/growth, timelines, scenarios/risks). Add a one-line deviation note if sections differ from PLAN.
+- ## S1..Sn (exactly as defined in PLAN) — each section answers its mapped sub-question and **integrates all mapped evidence**:
+  - Weave facts; where ≥3 related numbers exist, add a small Markdown table.
+  - **Integrate as much of the TRACE/TOOL_CALLS information as possible** in a structured way based on the question decomposition; if an item is only contextual, summarize briefly and attribute.
+  - Call out conflicts with both sources cited.
+- ## Recommendations — actionable, prioritized; must follow from cited evidence.
+- ## Conclusion — 3–6 sentences directly answering the QUESTION.
+- ## Sources used — deduplicated raw URLs, one per line (only those cited above).
+
+========================
+EXHAUSTIVENESS & COVERAGE
+========================
+- **Inclusion duty:**  Factual detail explored in TRACE must appear in the final report unless completely irrlevant.
+- **Do not compress away specifics.** Prioritize: (1) exact figures/dates, (2) named entities/products, (3) risks/criticisms, (4) methods/assumptions, (5) contextual detail.
+- **Numeric presentation:** For ≥3 related numbers, render a small Markdown table with citations.
+- Be verbose in the Main Body; detailed explanations / exhaustive covergage, novel synthesis, insights and dense citations are encouraged.
+
+========================
+QUALITY TARGETS (SCORING GUARDRAILS)
+========================
+- **Comprehensiveness (COMP):** Every URL/tool_response mapped in the plan is integrated. The REPORT should **strive to integrate maximum trace information** in context.
+- **Insight/Depth (DEPTH):** Use contrast/benchmarks, timelines, ratios/growth, causal links, scenarios, and risk framing to explain “why it matters,” building insights **on top of the existing evidence** (no new facts).
+- **Instruction-Following (INST):** Sections mirror sub-questions; each SQ is explicitly answered, the report should be precise and not digress from what is asked in the question. 
+- **Readability (READ):** Clear headings, short paragraphs, lead sentences with takeaways, tables for numeric clusters, and **dense-but-accurate** citations.
+
+========================
+STRICT OUTPUT FORMAT
+========================
+- You must give exactly one single output with the private planning / thinking enclosed within the <think></think> and the public facing report follwing that:
+   <think>[Plan here]</think>[Report here]
+- The REPORT is strictly public-facing (no meta/process/thinking).
+- Markdown only. Public-facing rationale; no hidden notes or menntion of the search trace or the thinking process in the report.
+- Target lengt for the Report Section: **≥2000 words** (longer if complexity requires).
+"""
+
+# SUMMARY_SYS_PROMPT: Final[str] = r"""
+# You are a Summary Assistant.
+
+# Goal: Produce a public-facing response that structures all information from input trace into a single answer.
+
+# ========================
+# INPUTS
+# ========================
+# - QUESTION: user's question.
+# - TRACE: transcript (assistant/user/tool snippets).
+# - TOOL_CALLS: raw tool calls (includes URLs and tool_responses).
+
+# ========================
+# RESPONSE (ANSWER) (MANDATORY CONTENT)
+# ========================
+# - The response to the user's question, enclosed in <answer></answer> tags.
+# - The response must be well-structured and detailed, covering all important steps, ideas, and any evidence/calculations found in the trace.
+# - If the task is CLOSED-ENDED (math/logic with a determinate result; factual single value/word; code producing a definite output), think and reason/plan internally and respond with the final part (explanation, method, proof, etc.) and present the result boxed with LaTeX: \boxed{…}.
+# - If the task is OPEN-ENDED (analysis, synthesis, design choices, multiple valid outcomes), think and reason/plan internally and respond containing a detailed explanation of the search trace, sources, investigation, process/methodology, result/outcome/solution, conclusion, etc.; i.e. create a nicely-structured and detailed structure of the answer for the question, that can be shown to the user who asked it.
+# - Keep the answer detailed and well-structured, providing a thorough explanation/methodology/solution for the final response, whatever is desired for in the user query. Do not just give a one-line/very-short final response. The answer maybe short if the question is trivial, but it must be well-structured and thorough.
+
+# ========================
+# STRICT OUTPUT FORMAT
+# ========================
+# - You must give exactly one single output with the private planning / thinking enclosed within the <think></think> and the public facing report follwing that:
+#    <think>[Plan here]</think><answer>[Final Answer here]</answer>
+# - The final answer is strictly public-facing (no meta/process/thinking).
+# - Markdown only.
+# """
+ 
+SUMMARY_SYS_PROMPT: Final[str] = r"""
+You are an expert search trace structurer. Given a QUESTION and the full search TRACE (may include tool-call notes),
+write a clear, accurate, self-contained explanation/solution using only the information in the trace. Do not add external facts.
+ 
+What to produce:
+- A single, readable well-structured narrative / solution that covers all important steps, ideas, and any evidence/calculations found in the trace.
+- If the task is CLOSED-ENDED (math/logic with a determinate result; factual single value/word; code producing a definite output), think and reason/plan internally and respond with the final part (explanation, method, proof, etc.) and present the result boxed with LaTeX: \boxed{…}.
+- If the task is OPEN-ENDED (analysis, synthesis, design choices, multiple valid outcomes), think and reason/plan internally and respond containing a detailed explanation of the search trace, sources, investigation, process/methodology, result/outcome/solution, conclusion, etc.; i.e. create a nicely-structured and detailed structure of the answer for the question, that can be shown to the user who asked it.
+- Note: The final part is strictly public-facing (no meta/process/thinking) and is to be enclosed in <answer></answer> tags and the thinking/planning/reasoning is internal and to be compulsorily enclosed within <think></think> tags.
+- The final part can be short or detailed depending on the question but has to be seperately enclosed in <answer></answer> tags and will be after the thinking block (which is to be enclosed in <think></think> tags).
+
+Style:
+- Clear prose and paragraphs; use LaTeX sparingly for clarity in math.
+- Prefer thorough and detailed coverage; keep it shorter for trivial items.
+- Use only facts present in the trace. If something is uncertain or missing, state it plainly and proceed with best-effort reasoning.
+- Provide detailed explanation/methodology/solution for the final response (public-facing part), whatever is desired for in the user query. Do not just give a one-line/very-short final response.
+- The final response should be well-structured and detailed and is to be enclosed within <answer></answer> tags.
+- The reasoning part is non-public facing and internal, and should be enclosed within <think></think> tags.
+
+**OUTPUT FORMAT:**
+- Enclose your thinking/reasoning/planning (if you are thinking before answering) within the <think></think> tags: <think>{thinking here}</think>{response here}
+- It is compulsory to use the <think></think> tags for enclosing planning/thinking/internal reasoning.
+- Return the final answer in the format: 
+```<think>{your thinking here}</think>
+<answer>{your final answer here}</answer>```
+- The final answer part of the response is strictly public-facing and should be well-structured and detailed.
+- Markdown only.
+"""

re_call/re_call.py

@@ -0,0 +1,1490 @@
+import re
+import os
+import json
+import requests
+import time
+from typing import List, Optional, Dict
+from .prompts import DEEPRESEARCH_SYS_PROMPT, SUMMARY_SYS_PROMPT 
+from functools import wraps
+from together import Together          # pip install together
+from datetime import datetime          # needed for retries / logging and date string (for giving current date and time to LLM)
+
+#     return decorator
+def retry(max: int = 10, sleep: int = 1, fallback=None):
+    """
+    Retry `max` times and, if still failing, return `fallback`
+    instead of raising.  This keeps outer loops alive.
+    """
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            for i in range(max):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    print(f"[retry] attempt {i+1}/{max} failed: {e}")
+                    if i == max - 1:                 # last try exhausted
+                        print(f"[retry] giving up – returning {fallback!r}")
+                        return fallback              # ← swallow the error
+                    if sleep:
+                        time.sleep(sleep)
+        return wrapper
+    return decorator
+
+class ReCall():
+    date_str = \
+    f"""
+
+    **Note**: Today's Date is {datetime.now().strftime("%Y-%m-%d")}, and time is {datetime.now().strftime("%H:%M:%S")}. This may be useful for answering questions about current events."""
+
+    anti_chinese_str = \
+    """
+    
+    **Note**: Do not respond in chinese, do not think in chinese, only think and respond/answer in English, unless explicitly instructed by the user to respond in some other language."""
+    
+    # proper_formatting_str = \
+    # """
+    # **Note**: Provide a well-structured answer first, then put only the final short answer in \\boxed{{}}.
+
+    # **How to format your response**
+    # - Write in clear English prose and use Markdown headings/bullets where helpful.
+    # - Give a detailed, in-depth explanation of the steps or facts used.
+    # - Use LaTeX only for short formulas/equations. For multi-line LaTeX, include line breaks (\\\\) or environments like \\begin{{align}} ... \\end{{align}} when genuinely helpful.
+    # - Do **not** wrap the whole response in LaTeX. Only the final short answer goes in \\boxed{{...}} on its own line at the end.
+
+    # **Examples**
+
+    # 1) **Simple fact question**  
+    # **Question:** What is the capital of India?  
+    # **Brief rationale:** India’s seat of government and primary national institutions are located in New Delhi.  
+    # **Final:** \\boxed{{New Delhi}}
+
+    # 2) **Quick calculation**  
+    # **Question:** Convert 68^\\circ F to Celsius.  
+    # **Approach:** Use C = (F - 32) \\times \\tfrac{{5}}{{9}}.  
+    # **Computation:** (68 - 32) \\times \\tfrac{{5}}{{9}} = 20.  
+    # **Final:** \\boxed{{20^\\circ C}}
+
+    # 3) **Search & synthesis (structured, detailed)**  
+    # **Question:** When did the EU’s GDPR go into effect?
+    
+    # **Complete Final Response:**
+    # '''**Key findings (evidence, concise):**  
+    # - **European Commission overview** states GDPR “applies from 25 May 2018.”  
+    # - **EUR-Lex (Regulation (EU) 2016/679), Article 99**: entered into force 20 days after publication in the OJ (2016), and **applies from 25 May 2018**.  
+    # - **EDPB FAQs/communications** reiterate that enforcement/application begins **25 May 2018**.
+
+    # **Cross-check & validation:**  
+    # - Independent primary sources (Commission portal and EUR-Lex) agree on the same application date. A supervisory body source (EDPB) corroborates.
+
+    # **Common pitfalls addressed:**  
+    # - Some secondary blogs list **24 May 2018**—this confuses the **last day before** applicability with the first day **of** applicability.  
+    # - “Entered into force” in **2016** (post-publication) is not the same as “application/effective for obligations,” which is **2018**.
+
+    # **Date normalization:**  
+    # - Normalize to an unambiguous calendar date and present in a clear format (e.g., “May 25, 2018”).
+
+    # **Conclusion:**  
+    # - The effective (application) date for GDPR obligations across the EU is the same in all Member States and is confirmed by multiple primary sources.
+
+    # **Final:** \\boxed{{May\ 25,\ 2018}}'''
+    # """
+        
+    # print(f"Date string:\n'{date_str}'")
+    
+    # proper_formatting_str = \
+    # """
+    # **DeepResearch Response Protocol**  
+    # Provide a comprehensive, decision-grade report first, then put only the short final answer in \\boxed{{}} on its own line at the very end.
+
+    # ---
+
+    # ## Mandatory Sections (in order)
+
+    # 1) **Executive Summary**  
+    # - 5–10 bullets capturing the direct answer, key numbers/dates, and the top implications.  
+    # - Include any material uncertainty (e.g., “moderate confidence due to limited primary data”).
+
+    # 2) **Problem Framing & Scope**  
+    # - One short paragraph restating the question, goals, and audience.  
+    # - Clarify interpretations, exclusions, and assumptions. Define key terms and acronyms.
+
+    # 3) **Method (Search & Validation Plan)**  
+    # - 5–8 bullets detailing how you searched and validated. Include:  
+    #     - **Source priority:** primary/official (laws, filings, standards, regulator notices) → reputable secondary (major outlets, respected orgs) → tertiary/background.  
+    #     - **Query strategy:** main queries and alternates (synonyms, regional spellings, technical names).  
+    #     - **Freshness policy:** prefer the most recent authoritative updates; when dates matter, distinguish **event date**, **publication/update date**, and **effective date**.  
+    #     - **Triangulation rule:** corroborate all key claims with ≥2 independent reputable sources (or 1 clear primary).  
+    #     - **Inclusion/Exclusion:** note discarded sources (paywalled, low quality, self-published without review) and why.  
+    #     - **Conflict resolution:** how disagreements will be weighed (mandate, jurisdiction, methodological rigor, recency).
+
+    # 4) **Evidence Ledger (Cited Facts)**  
+    # - 6–15 bullets. Each bullet is a **Fact Card**:  
+    #     - **Claim:** one-sentence fact.  
+    #     - **Evidence:** short quote/figure/line (paraphrase unless a short quote is essential).  
+    #     - **Source:** Publisher/Title — (Event Date if applicable) — Publish/Update Date — Access Date.  
+    #     - **Confidence:** High / Medium / Low.  
+    # - Group with mini-subheadings where helpful (e.g., “Official notices”, “Regulatory filings”, “Press coverage”).  
+    # - Explicitly flag contradictions.
+
+    # 5) **Timeline of Key Events**  
+    # - A compact, chronological list linking milestones to sources; include both event and publication dates where relevant.
+
+    # 6) **Data Extraction & Normalization** (as needed)  
+    # - Present important numbers in a small table (≤8 rows) with units, currency (ISO codes, e.g., **USD**), and rounding policy (state precision, e.g., “rounded to 2 decimals”).  
+    # - Perform any conversions or calculations and show formulas succinctly (LaTeX inline for short formulas, e.g., \\( C = (F-32)\\times\\tfrac{{5}}{{9}} \\); use \\begin{{align}}…\\end{{align}} for multi-step math).  
+    # - Specify timezones for dates/times when relevant.
+
+    # 7) **Comparative & Sensitivity Analysis** (if applicable)  
+    # - Contrast competing interpretations, options, or sources; note trade-offs.  
+    # - Include a brief sensitivity or scenario check if a key parameter could materially change the conclusion.
+
+    # 8) **Synthesis & Conclusion**  
+    # - 2–4 tight paragraphs that integrate the evidence, resolve conflicts, and explain *why* the conclusion follows.  
+    # - Be explicit about scope limits and residual uncertainties.
+
+    # 9) **Risks, Caveats & Unknowns**  
+    # - Bullet the major risks, data gaps, and what would most change the answer.  
+    # - Note any ethical, legal, or safety considerations.
+
+    # 10) **Recommendations / Next Steps** (if applicable)  
+    #     - Actionable items tailored to the user’s likely goal (e.g., verify with regulator X, monitor source Y weekly, collect dataset Z).
+
+    # 11) **Answer (one sentence)**  
+    #     - State the direct answer clearly with units/timezone as needed.
+
+    # 12) **Final**  
+    #     - Repeat only the short final answer inside \\boxed{{...}} with no extra words.
+
+    # 13) **Source Log (Audit Trail)**  
+    #     - A compact, reproducible list: *Title — Publisher/Author — (Event Date, if any) — Publish/Update Date — Access Date — URL*.  
+    #     - Prefer diverse, authoritative domains; avoid duplicates.
+
+    # ---
+
+    # ## Formatting & Quality Rules
+
+    # - Use clear English with Markdown headings and bullets; favor short paragraphs.  
+    # - Do **not** reveal inner monologue or hidden chain-of-thought; provide only public-facing rationale.  
+    # - Use LaTeX sparingly for math; do **not** wrap the entire response in LaTeX. Only the final short answer goes in \\boxed{{...}}.  
+    # - Always specify units, currency codes, and timezones when relevant.  
+    # - When listing ≥3 items or comparing options, include a small, focused table rather than long prose.  
+    # - If information is uncertain or contested, *quantify* the uncertainty (confidence labels or ranges) and state why.
+
+    # ---
+
+    # ## Depth & Completeness Expectations
+
+    # - **Complex/high-stakes queries**: Populate all sections thoroughly; provide triangulated citations and explicit conflict resolution.  
+    # - **Simple fact queries**: Keep Sections 3–9 concise (one to two lines each) but still cite at least one authoritative source.  
+    # - Strive for neutrality, reproducibility, and decision usefulness over verbosity.
+
+    # ---
+    # """
+    
+    proper_formatting_str = """"""
+    
+    sys_prompt_non_search = """You are a helpful assistant. You will answer the user's question based on your knowledge and reasoning ability. You do not have access to the internet or any external tools. Do not use search. Answer all questions yourself.""" + date_str + anti_chinese_str
+    
+    sys_prompt_websailor_start = """
+    You are a Web Information Seeking Master. Your task is to thoroughly seek the internet for information and provide accurate answers to questions. No matter how complex the query, you will not give up until you find the corresponding information.
+    In this environment you have access to a set of tools you can use to assist with the user query. 
+    You may perform multiple rounds of function calls. In each round, you can call one or more functions.
+
+    As you proceed, adhere to the following principles:
+
+    1. **Persistent Actions for Answers**: You will engage in many interactions, delving deeply into the topic to explore all possible aspects until a satisfactory answer is found.
+
+    2. **Repeated Verification**: Before presenting a Final Answer, you will **cross-check** and **validate the information** you've gathered to confirm its accuracy and reliability.
+
+    3. **Attention to Detail**: You will carefully analyze each information source to ensure that all data is current, relevant, and from credible origins."""
+    
+    sys_prompt_websailor = """
+    You are a Web Information Seeking Master. Your task is to thoroughly seek the internet for information and provide accurate answers to questions. No matter how complex the query, you will not give up until you find the corresponding information.
+    In this environment you have access to a set of tools you can use to assist with the user query. 
+    You may perform multiple rounds of function calls. In each round, you can call one or more functions.
+
+    As you proceed, adhere to the following principles:
+
+    1. **Persistent Actions for Answers**: You will engage in many interactions, delving deeply into the topic to explore all possible aspects until a satisfactory answer is found.
+
+    2. **Repeated Verification**: Before presenting a Final Answer, you will **cross-check** and **validate the information** you've gathered to confirm its accuracy and reliability.
+
+    3. **Attention to Detail**: You will carefully analyze each information source to ensure that all data is current, relevant, and from credible origins.
+
+
+
+    Here are available functions in JSONSchema format: \n```json\n{func_schemas}\n```
+
+    In your response, you need to first think about the reasoning process in the mind and then conduct function calling to get the information or perform the actions if needed. \
+    The reasoning process and function calling are enclosed within <think> </think> and <tool_call> </tool_call> tags. \
+    The results of the function calls will be given back to you after execution, \
+    and you can continue to call functions until you get the final answer for the user's question.
+
+    For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+    <tool_call>
+    {{"name": <function-name>, "arguments": <args-json-object>}}
+    </tool_call>
+    """ + date_str + anti_chinese_str + proper_formatting_str
+    
+    sys_prompt_websailor_deepseek = """
+    You are a Web Information Seeking Master. Your task is to thoroughly seek the internet for information and provide accurate answers to questions. No matter how complex the query, you will not give up until you find the corresponding information.
+    In this environment you have access to a set of tools you can use to assist with the user query. 
+    You may perform multiple rounds of function calls. In each round, you can call one or more functions.
+
+    As you proceed, adhere to the following principles:
+
+    1. **Persistent Actions for Answers**: You will engage in many interactions, delving deeply into the topic to explore all possible aspects until a satisfactory answer is found.
+
+    2. **Repeated Verification**: Before presenting a Final Answer, you will **cross-check** and **validate the information** you've gathered to confirm its accuracy and reliability.
+
+    3. **Attention to Detail**: You will carefully analyze each information source to ensure that all data is current, relevant, and from credible origins.
+
+
+
+    Here are available functions in JSONSchema format: \n```json\n{func_schemas}\n```
+
+    In your response, you need to first think about the reasoning process in the mind and then conduct function calling to get the information or perform the actions if needed. \
+    The reasoning process and function calling are enclosed within <think> </think> and <tool_calls_begin> <tool_calls_end> tags. \
+    The results of the function calls will be given back to you after execution, \
+    and you can continue to call functions until you get the final answer for the user's question. \
+    Finally, if you have got the answer, enclose it within \\boxed{{}} with latex format and do not continue to call functions, \
+    i.e., <think> Based on the response from the function call, I get the weather information. </think> The weather in Beijing on 2025-04-01 is \\[ \\boxed{{20C}} \\].
+    """ + date_str + anti_chinese_str + proper_formatting_str
+    
+    # sys_prompt_websailor_deepseek = """
+    #         You are a Web Information Seeking Master. Seek the internet thoroughly and provide accurate answers. You may use tools multiple times.
+
+    #         Principles:
+    #         1) Persistent Actions for Answers: explore deeply until you find satisfactory information.
+    #         2) Repeated Verification: cross-check and validate before the final answer.
+    #         3) Attention to Detail: ensure sources are current, relevant, and credible.
+
+    #         You have the following tools (JSONSchema):
+    #         ```json
+    #         {func_schemas}
+    #         Follow this EXACT tool-call I/O protocol.
+
+    #         TO CALL ONE OR MORE TOOLS:
+    #         Respond only with this block (no extra text before/after):
+    #         <｜tool▁call▁begin｜>function<｜tool▁sep｜>{tool_name}{args_json}
+    #         <｜tool▁call▁end｜>
+    #         ... (repeat <｜tool▁call▁begin｜>…<｜tool▁call▁end｜> for multiple tools)
+    #         <｜tool▁calls▁end｜><｜end▁of▁sentence｜>
+
+    #         HOW TOOL RESULTS ARRIVE:
+    #         I will send tool outputs back embedded inside a single user message, each wrapped like:
+    #         <tool_response>{one_tool_call_you_made}
+    #         {tool_return_text_or_json}
+    #         </tool_response>
+
+    #         WHAT TO DO NEXT:
+
+    #         If you still need info, emit another tool-calls block (same exact format).
+
+    #         If you have the final answer, output:
+    #         <answer> …your final answer… </answer>
+    #         and DO NOT call any more tools.
+
+    #         Important:
+
+    #         Do not expose your internal reasoning; keep thoughts private.
+
+    #         When emitting a tool-calls block, do not include any explanations, only the block specified above.
+
+    #         Arguments must be valid JSON.
+
+    #         Stop tokens to respect: <｜end▁of▁sentence｜>
+    #         """
+
+    system_prompt = """In this environment you have access to a set of tools you can use to assist with the user query. \
+    You may perform multiple rounds of function calls. \
+    In each round, you can call one or more functions. \
+
+    Here are available functions in JSONSchema format: \n```json\n{func_schemas}\n```
+
+    In your response, you need to first think about the reasoning process in the mind and then conduct function calling to get the information or perform the actions if needed. \
+    The reasoning process and function calling are enclosed within <think> </think> and <tool_call> </tool_call> tags. \
+    The results of the function calls will be given back to you after execution, \
+    and you can continue to call functions until you get the final answer for the user's question. You are encouraged to utilize as many function calls as possible. \
+    Finally, if you have got the answer, wrap it in <answer> </answer> **and do not call any more functions**, \
+    e.g. <think> Based on the tool results … </think> <answer>20 °C</answer>.
+
+    For each function call, return a JSON object with function name and arguments within <tool_call></tool_call> XML tags:
+    <tool_call>
+    {{"name": <function-name-1>, "arguments": <args-json-object>}}
+    </tool_call>""" + date_str + anti_chinese_str + proper_formatting_str
+
+    system_prompt_budget = """
+        You are an autonomous reasoning agent with access to external tools.
+
+        The conversation will retain only the *most-recent* <tool_response> block; older ones disappear.  
+        As soon as you receive tool results, extract the *essential facts tables links etc* that might be needed for later and restate them inside your <think> section.  
+         **Never copy large bodies of text** or raw JSON from tool output into your visible reply; summarise instead.
+
+        ◎ **Workflow**  
+        1. In every round, start with <think> … </think> to lay out your short reasoning.  
+        2. If you need external information or an action, emit one or more <tool_call> … </tool_call> blocks (JSON spec below).  
+        3. When the environment returns <tool_response>, continue reasoning; you may call more tools.  
+        4. Once you can answer the user, wrap the final result in <answer> … </answer> and STOP calling tools.
+
+        ◎ **Tool call format** (do **not** restate the schema or any explanations):  
+        <tool_call>
+        {{"name": <function-name-1>, "arguments": <args-json-object>}}
+        </tool_call>
+
+        Here are available functions in JSONSchema format: \n```json\n{func_schemas}\n```
+    """ + date_str + anti_chinese_str + proper_formatting_str
+
+
+
+    system_prompt_forcing_tool_call = """
+    In this environment you have access to a set of tools you can use to assist with the user query.
+    You may perform multiple rounds of function calls upto ten. In each round, you can call upto three functions.
+
+    ──────────────────────── AVAILABLE TOOLS ────────────────────────
+    ```json
+    [
+    {
+        "type": "function",
+        "function": {
+        "name": "pubmed_search",
+        "description": "Search PubMed for Medical related queries.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+            "query":  { "type": "string",  "description": "Query to search for." },
+            "top_n":  { "type": "integer", "description": "Number of hits", "default": 3 }
+            },
+            "required": ["query"]
+        }
+        }
+    }
+    ]
+    ```
+
+    ────────────────────────────── RULES ──────────────────────────────
+    1. You MUST issue one pubmed_search tool call for each answer choice. Each query must relate the clinical context to that option.
+    2. You MAY NOT skip any option or decide based only on internal reasoning. Evidence must be retrieved for all choices.
+    3. You MAY issue follow-up tool calls if your reasoning leads you to need more evidence.
+    4. You MUST wrap all reasoning in <think> </think> tags and all tool usage in <tool_call> </tool_call> tags. Number of <tool_call> and </tool_call> tokens in the entire trace MUST always match.
+    5. Do NOT casually emit the  <tool_call> </tool_call> during reasoning unless explicitly calling a tool in the proper format. 
+    5. Your final answer must be enclosed a single letter corresponding to the correct option enclosed in the <answer> </answer> tags. Do not output anything else inside these tags.
+    6. DO NOT use any other confusing tags like <thiking> or </thinking>. 
+    7. Each <think> </think> block MUST be followed by a <tool_call> </tool_call> or <answer> </answer> or else the program will break without an answer.
+
+    ───────────────────── DUMMY EXAMPLE INTERLEAVED SKELETON ─────────────────────
+    <think>
+    We are presented with a 54-year-old woman with invasive ductal carcinoma of the breast and osteolytic lesions in the thoracic spine. This strongly suggests metastatic spread. Our task is to determine the most likely anatomical route of metastasis to the spine.
+
+    Let’s examine the given options:
+    A. Hemiazygos vein  
+    B. Posterior intercostal veins  
+    C. Batson’s vertebral venous plexus  
+    D. Internal mammary lymphatics
+
+    We'll evaluate each option in turn using available literature and known anatomical pathways.
+    **Option A: Hemiazygos vein**  
+    We begin by evaluating whether the hemiazygos vein could be involved in metastatic spread from breast cancer to the spine.
+    </think>
+    <tool_call>
+    {"name": "pubmed_search", "arguments": {"query": "breast cancer metastasis hemiazygos vein", "top_n": 2}}
+    </tool_call>
+    <tool_response>
+    ...
+    </tool_response>
+    <think>
+    There is limited or no strong evidence suggesting the hemiazygos vein is a common or primary route for vertebral metastasis from breast cancer.
+    Lets explore **Option B: Posterior intercostal veins**  and  **Option C: Batson’s vertebral venous plexus** and **Option D:Internal mammary lymphatics**
+    </think>
+    <tool_call>
+    {"name": "pubmed_search", "arguments": {"query": "posterior intercostal veins breast cancer spinal metastasis", "top_n": 3}}
+    </tool_call>
+    <tool_call>
+    {"name": "pubmed_search", "arguments": {"query": "Batson vertebral venous plexus breast cancer metastasis", "top_n": 3}}
+    </tool_call>
+    <tool_call>
+    {"name": "pubmed_search", "arguments": {"query": "Internal mammary lymphatics breast cancer metastasis", "top_n": 3}}
+    </tool_call>
+    <tool_response>
+    ...
+    </tool_response>
+    <think>
+    While the posterior intercostal veins may be involved in venous drainage, there is insufficient evidence to support them as a primary route for metastasis to the vertebral column.
+    where as Batson’s vertebral venous plexus — a valveless venous network that connects the thoracic and abdominal veins directly to the spine. I to find more specific information about option C.
+    </think>
+    <tool_call>
+    {"name": "pubmed_search", "arguments": {"query": ""Batson vertebral venous plexus breast cancer metastasis in people over 50", "top_n": 1}}
+    </tool_call>
+     <think>
+    After evaluating all four options, the most plausible route for breast cancer metastasis to the thoracic spine is clearly via  Batson’s vertebral venous plexus:
+    </think>
+    <answer>C</answer>
+    """ + date_str + anti_chinese_str + proper_formatting_str
+    # STOP_TOKENS =STOP_TOKENS = ["<|im_end|>", "<|endoftext|>"
+
+
+    def __init__(self, executor_url):
+        self.executor_url = executor_url
+
+    def init_prompt(self, func_schemas, question, old_prompt: Optional[str] = None, search_on: bool = True) -> str:
+        if old_prompt is None or len(old_prompt.strip()) == 0:
+            if search_on:
+                system_prompt = f"<|im_start|>system\n{self.sys_prompt_websailor.format(func_schemas=func_schemas)}<|im_end|>"
+            else:
+                system_prompt = f"<|im_start|>system\n{self.sys_prompt_non_search}<|im_end|>"
+            user_prompt = f"<|im_start|>user\n{question}<|im_end|>"
+            assistant_prefix = f"<|im_start|>assistant\n<think>"
+            return system_prompt + "\n" + user_prompt + "\n" + assistant_prefix
+        else:
+            user_prompt = f"<|im_start|>user\n{question}<|im_end|>"
+            assistant_prefix = f"<|im_start|>assistant\n<think>"
+            return old_prompt + "\n" + user_prompt + "\n" + assistant_prefix
+        
+    def replace_sys_prompt(self, old_prompt: str, func_schemas: str, search_on: bool = True) -> str:
+        if search_on:
+            new_sys_prompt = f"<|im_start|>system\n{self.sys_prompt_websailor.format(func_schemas=func_schemas)}<|im_end|>"
+            old_sys_prompt = f"<|im_start|>system\n{self.sys_prompt_non_search}<|im_end|>"
+        else:
+            new_sys_prompt = f"<|im_start|>system\n{self.sys_prompt_non_search}<|im_end|>"
+            old_sys_prompt = f"<|im_start|>system\n{self.sys_prompt_websailor.format(func_schemas=func_schemas)}<|im_end|>"
+            
+        return old_prompt.replace(old_sys_prompt, new_sys_prompt)
+
+    def _strip_old_tool_responses(self, prompt: str) -> str:
+        TOOL_RESPONSE_RE = re.compile(r"<tool_response>.*?</tool_response>\s*", re.DOTALL)
+        """Remove every existing <tool_response> … </tool_response> block."""
+        return TOOL_RESPONSE_RE.sub("", prompt)
+
+    def cat_assistant_response(self, curr_prompt, assistant_response):
+        return curr_prompt + assistant_response + "<|im_end|>"
+    
+    def cat_tool_results(self, curr_prompt, tool_calls, results):
+        tool_response_str = ""
+        for tool_call, result in zip(tool_calls, results):
+            tool_response_str += f"<tool_response>{tool_call}\n{result}\n</tool_response>\n"
+        tool_response_str = f"<|im_start|>user\n{tool_response_str}<|im_end|>"
+        assistant_prefix = f"<|im_start|>assistant\n<think>"
+        return curr_prompt + "\n" + tool_response_str + "\n" + assistant_prefix
+
+    def format_tool_call(self, tool_call_str: str):
+        """Convert JSON function call description to Python executable code string."""
+        try:
+            call_json = json.loads(tool_call_str)
+            func_name = call_json['name']
+            arguments = call_json.get('arguments', {})
+            
+            args_str = ', '.join(f"{k}={repr(v)}" for k, v in arguments.items())
+            return f"{func_name}({args_str})"
+        except Exception as e:
+            return f"Parse tool call failed: {e}"
+    
+    def execute_tool_calls(self, env: str, tool_calls: List[str]) -> List[str]:
+        def exe_tool_call(env, call):
+            url = self.executor_url + '/execute'
+
+            call_str = self.format_tool_call(call)
+            # print(call_str)
+            if call_str.startswith("error: parse tool call failed"):
+                return call_str
+
+            try:
+                data = {
+                    'env': env,
+                    'call': call_str
+                }
+                response = requests.post(url, json=data, timeout=60)
+                if response.status_code != 200:
+                    return f"error: {response.status_code}"
+                response = response.json()
+                ret_str = ''
+                if response['result']:
+                    ret_str += f'result: \n{response["result"]}\n'
+                if response['output']:
+                    ret_str += f'output: \n{response["output"]}\n'
+                if response['error']:
+                    ret_str += f'error: \n{response["error"]}\n'
+                return ret_str.strip()
+            except requests.exceptions.Timeout:
+                return "error: execution timed out"
+            except Exception as e:
+                return str(e)
+        
+        results = []
+        for tool_call in tool_calls:
+            result = exe_tool_call(env, tool_call)
+            results.append(result)
+        return results
+    
+    def validate_tool_calls(self, output_str):
+        start_tags = re.findall(r'<tool_call>', output_str)
+        end_tags = re.findall(r'</tool_call>', output_str)
+        
+        if len(start_tags) != len(end_tags):
+            return False
+            
+        start_positions = [m.start() for m in re.finditer(r'<tool_call>', output_str)]
+        end_positions = [m.start() for m in re.finditer(r'</tool_call>', output_str)]
+        
+        for start, end in zip(start_positions, end_positions):
+            if start >= end:
+                return False
+                
+        return True
+
+    def extract_tool_calls(self, output_str):
+        if not self.validate_tool_calls(output_str):
+            return []
+
+        try:
+            pattern = r'<tool_call>((?:(?!</tool_call>).)*)</tool_call>'
+            matches = re.finditer(pattern, output_str, re.DOTALL)
+            
+            return [match.group(1).strip() for match in matches]
+        except Exception as e:
+            return []
+        
+    def extract_tool_calls_deepseek(self, output_str):
+        if not self.validate_tool_calls(output_str):
+            return []
+
+        try:
+            pattern = r'<tool_calls_begin>((?:(?!</tool_calls_end>).)*)<tool_calls_end>'
+            matches = re.finditer(pattern, output_str, re.DOTALL)
+            
+            return [match.group(1).strip() for match in matches]
+        except Exception as e:
+            return []
+
+
+
+    @retry(max=5, sleep=1, fallback={"score": 0}) 
+    def run_ii_searcher(
+        self, 
+        env: str, 
+        func_schemas: str,
+        question: str,
+        tokenizer,
+        model_url="http://0.0.0.0:1214",
+        temperature: float = 0.0,
+        max_new_tokens: int = 40960,
+        ):
+        curr_prompt = self.init_prompt(func_schemas, question)
+        all_tool_calls= []
+    
+        for _ in range(16):
+            prompt_tokens = tokenizer(curr_prompt, return_tensors=None, add_special_tokens=False)["input_ids"]
+            max_tokens_left = max_new_tokens - len(prompt_tokens) - 100
+            # for oss model served via vllm
+            # response = requests.post(
+            #     f'{model_url}/v1/chat/completions', 
+            #     json={
+            #         "text": curr_prompt,
+            #         # "reasoning": "medium"
+            #         },
+            # ).json()
+            # for sglang served models hf models
+            response = requests.post(
+                f'{model_url}/generate', 
+                json={
+                    "text": curr_prompt,
+                    "sampling_params": {
+                        "temperature": temperature,
+                        "max_new_tokens": max_tokens_left,
+                        "repetition_penalty": 1.05
+                    },
+
+                }
+            ).json()
+            if "error" in response.keys():
+                print("resp",response)
+            curr_prompt = self.cat_assistant_response(curr_prompt, response['text'])
+
+            tool_calls: List[str] = self.extract_tool_calls(response['text'])
+            all_tool_calls += tool_calls
+
+            if len(tool_calls) == 0:
+                break
+    
+            else:
+                results: List[str] = self.execute_tool_calls(env, tool_calls)
+                curr_prompt = self.cat_tool_results(curr_prompt, tool_calls, results)
+
+        return curr_prompt, all_tool_calls
+    
+    # @retry(max=5, sleep=1, fallback={"score": 0}) 
+    # def run(
+    #     self, 
+    #     env: str, 
+    #     func_schemas: str,
+    #     question: str,
+    #     tokenizer,
+    #     model_url="http://0.0.0.0:1214",
+    #     temperature: float = 0.0,
+    #     max_new_tokens: int = 40960,
+    #     ):
+    #     curr_prompt = self.init_prompt(func_schemas, question)
+    #     all_tool_calls= []
+    
+    #     for i in range(32):
+    #         prompt_tokens = tokenizer(curr_prompt, return_tensors=None, add_special_tokens=False)["input_ids"]
+    #         max_tokens_left = max_new_tokens - len(prompt_tokens) - 100
+    #         # for oss model served via vllm
+    #         # response = requests.post(
+    #         #     f'{model_url}/v1/chat/completions', 
+    #         #     json={
+    #         #         "text": curr_prompt,
+    #         #         # "reasoning": "medium"
+    #         #         },
+    #         # ).json()
+    #         # for sglang served models hf models
+    #         response = requests.post(
+    #             f'{model_url}/generate', 
+    #             json={
+    #                 "text": curr_prompt,
+    #                 "sampling_params": {
+    #                     "temperature": temperature,
+    #                     "max_new_tokens": max_tokens_left,
+    #                     "repetition_penalty": 1.05
+    #                 },
+
+    #             }
+    #         ).json()
+    #         if "error" in response.keys():
+    #             print("resp",response)
+    #         curr_prompt = self.cat_assistant_response(curr_prompt, response['text'])
+
+    #         tool_calls: List[str] = self.extract_tool_calls(response['text'])
+    #         all_tool_calls += tool_calls
+
+    #         if len(tool_calls) == 0:
+    #             break
+    
+    #         else:
+    #             # print(f"Step-{i+1}")
+    #             results: List[str] = self.execute_tool_calls(env, tool_calls)
+    #             curr_prompt = self.cat_tool_results(curr_prompt, tool_calls, results)
+
+    #     return curr_prompt, all_tool_calls
+    from typing import List, Dict, Any, Tuple
+    import requests
+    
+    def build_summary_prompt(self, question: str, transcript: str, tool_calls: Any) -> str:
+        """Assemble a compact but detailed prompt for summarization."""
+        tool_str = ""
+        if tool_calls is not None:
+            try:
+                tool_str = str(tool_calls)
+            except Exception:
+                tool_str = "<unprintable tool_calls>"
+        return (
+            "You are given a DeepSearch investigation trace.\n\n"
+            f"Question:\n{question}\n\n"
+            "Trace (model transcript):\n"
+            f"{transcript}\n\n"
+            "Tool Calls (as-recorded):\n"
+            f"{tool_str}\n\n"
+            "— End of trace —"
+        )
+        
+    def reformat_trace(self, s: str) -> str:
+        if not s:
+            return s
+
+        t = s
+
+        # 1) Speaker tags: <|im_start|>assistant -> "ASSISTANT:\n"
+        def _speaker(m: re.Match) -> str:
+            role = (m.group(1) or "").strip().upper()
+            return f"\n{role}:\n"
+        t = re.sub(r"<\|im_start\|\>(\w+)", _speaker, t, flags=re.IGNORECASE)
+
+        # 2) End-of-message tag: drop but keep spacing
+        t = re.sub(r"<\|im_end\|\>", "\n", t, flags=re.IGNORECASE)
+
+        # 3) THINK blocks: replace tags with label, keep content
+        t = re.sub(r"<think\s*>", "", t, flags=re.IGNORECASE)
+        t = re.sub(r"</think\s*>", "\n", t, flags=re.IGNORECASE)
+
+        # 4) TOOL RESPONSE blocks: support both 'response' and the misspelt 'resonse'
+        t = re.sub(r"<tool_respon[sc]e\s*>", "SEARCH RESULT\n", t, flags=re.IGNORECASE)
+        t = re.sub(r"</tool_respon[sc]e\s*>", "\n", t, flags=re.IGNORECASE)
+
+        # 5) TOOL CALL wrappers: drop tags, keep the JSON/content
+        t = re.sub(r"</?tool_call\s*>", "", t, flags=re.IGNORECASE)
+
+        # 6) Any remaining ChatML specials like <|eot_id|>, <|...|> -> remove
+        t = re.sub(r"<\|[^>]+?\|>", "", t)
+
+        # 7) Remove any other angle-bracket tags we didn’t explicitly keep
+        #    (leaves inner text intact). This will strip e.g. <tool_response_extra>
+        t = re.sub(r"</?[^>\n]+?>", "", t)
+
+        # 8) Normalize whitespace (collapse 3+ newlines to 2)
+        t = re.sub(r"\n{3,}", "\n\n", t).strip()
+
+        return t
+    
+    def _openai_client(self):
+        try:
+            from openai import OpenAI  # type: ignore
+        except Exception as e:
+            raise RuntimeError("openai package not installed. `pip install openai`") from e
+        return OpenAI()
+    
+    def init_summary_prompt(self, system_prompt: str, prompt: str) -> str:
+        system_prompt = f"<|im_start|>system\n{system_prompt}<|im_end|>"
+        user_prompt = f"<|im_start|>user\n{prompt}<|im_end|>"
+        assistant_prefix = f"<|im_start|>assistant\n<think>"
+        return system_prompt + "\n" + user_prompt + "\n" + assistant_prefix
+    
+    def _call_hf_endpoint(self, base_url: str, system_prompt: str, prompt: str, temperature: float, max_tokens: int, deepresearch_on: bool) -> str:
+        curr_prompt = self.init_summary_prompt(system_prompt, prompt)
+        
+        hf_token= os.environ['HF_TOKEN']
+
+        headers = {
+            "Accept" : "application/json",
+            "Authorization": f"Bearer {hf_token}",
+            "Content-Type": "application/json" 
+        }
+
+        # print(f"User Prompt:\n{curr_prompt}\n\n")
+
+        response_summary = requests.post(
+            url=f"{base_url}",
+            headers=headers,
+            json={
+                "inputs": curr_prompt,
+                "parameters": {
+                    "temperature": temperature,
+                    "max_new_tokens": max_tokens,
+                    "top_p": 0.95,
+                    "repetition_penalty": 1.05,
+                },
+            },
+            timeout=300,
+        ).json()
+        
+        if isinstance(response_summary, list):
+            response_summary = response_summary[0]
+
+        if isinstance(response_summary, dict) and "error" in response_summary:
+            # Log the error as assistant text for visibility and break
+            err_msg = f"[model_error] {response_summary.get('error')}"
+            print("Got error response from summarising model:", err_msg, end="\n\n")
+
+        assistant_text = response_summary.get("generated_text", "")
+        
+        if curr_prompt == assistant_text[:len(curr_prompt)]:
+                assistant_text = assistant_text[len(curr_prompt):]
+                
+        # print(assistant_text)
+        
+        report = re.split(r"</think\s*>", assistant_text, flags=re.IGNORECASE)[-1]
+        # plan = re.split(r"</think\s*>", assistant_text, flags=re.IGNORECASE)[0]
+        
+        # print(report, "\n\n")
+        
+        if not deepresearch_on:
+            report = report.strip()
+            # report = report[::-1]
+            # str_find = "Final Answer:"
+            # pos = report.find(str_find[::-1])
+            # pos += len(str_find)
+            # report = report[pos:][::-1]
+            # report = report.rstrip('# \n-').strip(' \n-')
+            
+            start_tag = "<answer>"
+            end_tag = "</answer>"
+            pos_start = report.find(start_tag)
+            pos_end = report[pos_start:].find(end_tag) + pos_start
+            answer = report
+            if pos_start != -1 and pos_end != -1:
+                answer = report[pos_start + len(start_tag):pos_end].strip()
+            
+            str_find = "Final Answer:"
+            if str_find in answer:
+                answer = answer[::-1]
+                pos = answer.find(str_find[::-1])
+                pos += len(str_find)
+                answer = answer[pos:][::-1]
+                answer = answer.rstrip('# \n-').strip(' \n-')
+            
+            # print("answer:")
+            # print(answer, "\n\n")
+            
+            return answer
+        
+        report = report.strip()
+        report = report[::-1]
+        str_find = "Sources used"
+        pos = report.find(str_find[::-1])
+        pos += len(str_find)
+        report = report[pos:][::-1]
+        report = report.rstrip('# \n-').strip(' \n-')
+        
+        if not report.startswith("##") and report.startswith("#"):
+            report = "#" + report
+        elif not report.startswith("##") and not report.startswith("#"):
+            report = "## " + report
+        
+        # report = '\n\n' + report.strip()
+        
+        # print(report.find('Executive Summary'), report.find('#'))
+        # print(f"'{report[:20]}'")
+        
+        # print(report,"\n\n")
+        
+        urls = {}
+        count = 1
+        
+        while "[http" in report:
+            start_idx = report.find("[http")
+            end_idx = report.find("]", start_idx)
+            if end_idx != -1:
+                url_string = report[start_idx + 1:end_idx]
+                url_list = []
+                while len(url_string) > 0:
+                    pos1 = url_string.find(";")
+                    pos2 = url_string.find(",")
+                    pos3 = url_string.find(" ")
+                    
+                    if pos1 == -1:
+                        pos1 = len(url_string) + 1
+                    if pos2 == -1:
+                        pos2 = len(url_string) + 1
+                    if pos3 == -1:
+                        pos3 = len(url_string) + 1
+                    
+                    pos = min(pos1, pos2, pos3)
+                    
+                    if pos == len(url_string) + 1:
+                        url = url_string
+                    else:
+                        url = url_string[:pos]
+                    
+                    url_list.append(url)
+
+                    if pos < len(url_string):
+                        url_string = url_string[pos + 1:].lstrip(" ,;")
+                    else:
+                        break
+                
+                report_new = report[:start_idx] + '(**'
+                for url in url_list:
+                    if url not in urls:
+                        urls[url] = count
+                        count += 1
+                    report_new += f'[{urls[url]}], '
+                report_new = report_new[:-2]
+                report_new += '**)' + report[end_idx+1:]
+                report = report_new
+            else:
+                break
+            
+        if len(urls) > 0:
+            report += "\n\n## Sources used:\n"
+            sorted_urls = sorted(urls.items(), key=lambda x: x[1])
+            for url, idx in sorted_urls:
+                report += f"- **{idx}**: {url}\n"
+            report += '\n'
+            # adding references (auto-removed in markdown)
+            for url, idx in sorted_urls:
+                report += f"[{idx}]: {url}\n"
+                
+        # print(report,"\n\n")
+        
+        return report
+    
+    def _route_and_summarize(
+        self,
+        summary_llm: str,
+        system_prompt: str,
+        prompt: str,
+        *,
+        temperature: float,
+        max_tokens: int,
+        deepresearch_on: bool,
+    ) -> str:
+        """
+        If `summary_llm` starts with 'http', treat as vLLM base_url; else treat as an OpenAI model id.
+        For vLLM, prepend [SYSTEM]/[USER] tags; for OpenAI, pass messages with system+user.
+        """
+        if not summary_llm.strip().lower().startswith("gpt-"):
+            # print(system_prompt)
+            # print(prompt)
+            return self._call_hf_endpoint(summary_llm, system_prompt, prompt, temperature=temperature, max_tokens=max_tokens, deepresearch_on=deepresearch_on)
+
+        else:
+            client = self._openai_client()
+            rsp = client.chat.completions.create(
+                model=summary_llm,
+                temperature=temperature,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user",   "content": prompt},
+                ],
+                max_tokens=max_tokens,
+            )
+            
+            return rsp.choices[0].message.content or ""
+
+    @retry(max=5, sleep=1, fallback={"score": 0})
+    def run(
+        self, 
+        env: str, 
+        func_schemas: str,
+        question: str,
+        tokenizer,
+        model_url: str = "http://0.0.0.0:1214",
+        temperature: float = 0.0,
+        max_new_tokens: int = 40960,
+        top_p: float = 0.6,
+        old_prompt: Optional[str] = None,
+        deepresearch_on: bool = True,
+        summary_llm: str = "gpt-4.1-mini"
+    ):
+        # ) -> Tuple[str, List[str], List[Dict[str, str]]]:
+        """
+        Returns:
+            curr_prompt: the final prompt buffer (with assistant/tool traces you maintain internally)
+            all_tool_calls: flat list of all tool call strings extracted across steps
+            chat: a lightweight chat transcript list[{"role": "...", "content": "..."}]
+                • 'user' items = the original question + aggregated tool responses
+                • 'assistant' items = model responses (and a compact line-list of tool calls)
+        """
+        # off_str = "\n\n**User has TURNED OFF search**. **DO NOT use search**. **Answer all questions YOURSELF**. **DO NOT use any tools**.\n**YOUR FIRST-RESPONSE WILL BE CONSIDERED AS THE FINAL ANSWER**. **YOU WILL NOT GET TO CALL TOOLS AND WAIT FOR TOOL RESULTS AND THEN ANSWER**.\n**YOU WON'T BE ALLOWED TO CHAT AND CALL TOOLS, IN A MULTI-TURN FASHION**. **YOU WILL CHAT IN A SINGLE-TURN FORMAT**.\n**SO MAKE SURE YOUR FIRST RESPONSE IS THE FINAL ANSWER**.\n"
+        
+        # if not search_on and (old_prompt is not None and self.sys_prompt_websailor_start not in old_prompt):
+        #     question += off_str
+        
+        search_on = True
+        
+        if old_prompt is not None:
+            old_prompt = self.replace_sys_prompt(old_prompt, func_schemas, search_on)
+        
+        # Build runtime prompt and initialize accumulators
+        curr_prompt = self.init_prompt(func_schemas, question, old_prompt, search_on)
+        all_tool_calls: List[str] = []
+        chat: List[Dict[str, str]] = []
+
+        # Seed transcript with JUST the question (no system prompt)
+        chat.append({"role": "user", "content": question})
+        
+        for i in range(64):
+            # Budget tokens for this step
+            prompt_tokens = tokenizer(curr_prompt, return_tensors=None, add_special_tokens=False)["input_ids"]
+            max_tokens_left = max(1, max_new_tokens - len(prompt_tokens) - 100)
+
+            # ---- Model call (sglang/vLLM-style JSON) ----
+            # If you switch to /v1/chat/completions, adjust accordingly.
+            hf_token= os.environ['HF_TOKEN']
+
+            headers = {
+                "Accept" : "application/json",
+                "Authorization": f"Bearer {hf_token}",
+                "Content-Type": "application/json" 
+            }
+
+            # print(f"User Prompt:\n{curr_prompt}\n\n")
+
+            response = requests.post(
+                url=f"{model_url}",
+                headers=headers,
+                json={
+                    "inputs": curr_prompt,
+                    "parameters": {
+                        "temperature": temperature,
+                        "max_new_tokens": max_tokens_left,
+                        "top_p": top_p,
+                        "repetition_penalty": 1.05,
+                    },
+                },
+                timeout=300,
+            ).json()
+            
+            if isinstance(response, list):
+                response = response[0]
+
+            if isinstance(response, dict) and "error" in response:
+                # Log the error as assistant text for visibility and break
+                err_msg = f"[model_error] {response.get('error')}"
+                print("Got error response from model:", err_msg, end="\n\n")
+                chat.append({"role": "assistant", "content": err_msg})
+                break
+
+            assistant_text = response.get("generated_text", "")
+            
+            if curr_prompt == assistant_text[:len(curr_prompt)]:
+                # print("Current prompt is a prefix to generated text.")
+                # If the assistant's response is just a continuation of the prompt, we can use it directly
+                assistant_text = assistant_text[len(curr_prompt):]
+
+            # print(f"Assistant Text:\n{assistant_text}\n\n")
+
+            # Append assistant's raw text to chat
+            chat.append({"role": "assistant", "content": assistant_text})
+
+            # Update your running prompt with assistant text
+            curr_prompt = self.cat_assistant_response(curr_prompt, assistant_text)
+
+            # Extract tool calls from the assistant text
+            if search_on:
+                tool_calls: List[str] = self.extract_tool_calls(assistant_text)
+            
+            else:
+                tool_calls: List[str] = []
+
+            # yield "assistant_resp", (assistant_text, tool_calls)
+
+            if tool_calls:
+                yield "assistant_resp", (assistant_text, tool_calls)
+                all_tool_calls.extend(tool_calls)
+
+                # Log tool calls as an assistant message (newline-joined)
+                chat.append({"role": "assistant", "content": "\n".join(tool_calls)})
+
+                # Execute tools and collect results
+                results: List[str] = self.execute_tool_calls(env, tool_calls)
+                
+                yield "tool_results", (results, )
+
+                # Feed tool results back into prompt
+                curr_prompt = self.cat_tool_results(curr_prompt, tool_calls, results)
+
+                # Aggregate tool responses into a single user message
+                tool_res_blocks = []
+                for idx, (call, res) in enumerate(zip(tool_calls, results), 1):
+                    tool_res_blocks.append(f"[Tool {idx}] Result:\n{res}")
+                chat.append({"role": "user", "content": "\n\n".join(tool_res_blocks)})
+
+            else:
+                if search_on:
+                    prompt = self.build_summary_prompt(question, self.reformat_trace(curr_prompt) or "", all_tool_calls)
+                    system_prompt = DEEPRESEARCH_SYS_PROMPT if deepresearch_on else SUMMARY_SYS_PROMPT
+
+                    summary_text = self._route_and_summarize(
+                        summary_llm=summary_llm if deepresearch_on else model_url,
+                        system_prompt=system_prompt,
+                        prompt=prompt,
+                        temperature=0.6,
+                        max_tokens=16000,
+                        deepresearch_on=deepresearch_on
+                    )
+                    
+                    summary_text_splits = summary_text.split("</think>")
+                    summary_text_initial = summary_text_splits[0]
+                    summary_text_initial = summary_text_initial.replace("<think>", "").strip()
+                    summary_text_final = summary_text_splits[-1]
+                    
+                    if len(summary_text_initial) > 0 and "</think>" in summary_text:
+                        yield "assistant_resp", (summary_text_initial, [])
+                        yield "tool_results", ([], )
+                    yield "assistant_resp", (summary_text_final, tool_calls)
+                    # print(f"No tool calls found in assistant response.\nAssistant Response:\n{assistant_text}\n\n")
+                else:
+                    yield "assistant_resp", (assistant_text, tool_calls)
+                    print(f"Search is off, so no tool calls expected and no tool calls called.\nAssistant Response:\n{assistant_text}\n\n")
+                # No tool calls → model produced a final answer; stop.
+                break
+
+        # Return the original outputs plus the chat-style transcript
+        # return curr_prompt, all_tool_calls, chat
+        
+        return "end", (curr_prompt, )
+
+    @retry(max=5, sleep=1, fallback={"score": 0})
+    def run_deepseek(
+        self,
+        env: str,
+        func_schemas: str,
+        question: str,
+        model_name: str,
+        temperature: float = 0.0,
+        top_p: float = 0.95,
+        max_tokens: int = 32768,
+    ):
+        # print("AA"* 100)
+        """
+        Chat-based ReCall loop for DeepSeek-R1 on Together.
+        """
+        sys_content = self.sys_prompt_websailor_deepseek.format(func_schemas=func_schemas)
+        # sys_content = self.init_prompt(func_schemas, question)
+
+        messages = [
+            {"role": "system", "content": sys_content},
+            {"role": "user",   "content": question},
+        ]
+    
+        # client = Together(api_key="")
+        client = Together(api_key="")
+        all_tool_calls = []
+        for turn in range(32):  # up to 10 reasoning turns
+            resp = client.chat.completions.create(
+                model=model_name,
+                # model="Qwen/Qwen3-235B-A22B-fp8-tput", 
+                messages=messages,
+                temperature=temperature,
+                top_p=top_p,
+                max_tokens=39000,
+                stop=["<｜end▁of▁sentence｜>", "<|im_end|>"]
+            )
+            # print(resp)
+            
+
+            assistant_text = resp.choices[0].message.content
+            # print(assistant_text)
+            messages.append({"role": "assistant", "content": assistant_text})
+            # print(f"assistant_output: {assistant_text}")
+
+            # ⛑ Safe tool call extraction with diagnostic
+            # try:
+            # print("Extracting tool calls")
+            tool_calls = self.extract_tool_calls_deepseek(assistant_text)
+            print(tool_calls)
+            all_tool_calls += tool_calls
+            # except Exception as e:
+            #     print(f"Extraction failed with exception {e}")
+            #     err_msg = f"<tool_response>Tool call extraction failed on turn {turn+1}: {str(e)}</tool_response>"
+            #     messages.append({"role": "user", "content": err_msg})
+            #     continue  # continue to next turn instead of breaking
+            if "<answer>" in assistant_text:
+                break
+    
+            if len(tool_calls) != 0:
+                results = self.execute_tool_calls(env, tool_calls)
+                tool_resp_block = "".join(
+                    f"<tool_response>{c}\n{r}\n</tool_response>\n"
+                    for c, r in zip(tool_calls, results)
+                )
+                messages.append({"role": "user", "content": tool_resp_block})
+                # print(f"Tool Response {tool_resp_block}")
+            else:
+                print("no answer or tool call")
+                break
+
+        trajectory = "\n".join(
+            f"<{m['role']}>\n{m['content']}" for m in messages
+            if m["role"] != "system"
+        )
+        return trajectory, all_tool_calls
+    
+    
+         # ────────────────────────────────────────────────────────────────
+    # HF-endpoint version of “retrieve → inject → tool loop”
+    # ────────────────────────────────────────────────────────────────
+    @retry(max=5, sleep=1, fallback=None)
+    def run_with_prompt_injection(
+        self,
+        env: str,
+        func_schemas: str,
+        question: str,
+        model_url: str = "http://0.0.0.0:1214",
+        temperature: float = 0.0,
+        max_new_tokens: int = 512,
+        top_n: int = 5,
+    ):
+        """
+        0) call pubmed_search(question, top_n) once via the sandbox
+        1) inject those snippets into the very first user message
+        2) continue with the normal multi-turn ReCall loop against *model_url*
+        """
+
+        # 0️⃣ do a single retrieval tool call
+        retrieve_call = json.dumps({
+            "name": "pubmed_search",
+            "arguments": {"query": question, "top_n": top_n}
+        })
+        retrieval_raw = self.execute_tool_calls(env, [retrieve_call])[0]
+        try:
+            snippets_block = retrieval_raw.split("result:", 1)[-1].strip()
+        except Exception:
+            snippets_block = ""
+
+        # 1️⃣ build initial prompt with injected snippets
+        user_msg = (
+            f"Question: {question}\n\n"
+            "Here are some relevant PubMed snippets:\n"
+            f"{snippets_block}"
+        ) if snippets_block else f"Question: {question}"
+
+        sys_prompt = self.init_prompt(func_schemas, question)
+        system_prompt = f"<|im_start|>system\n{sys_prompt}<|im_end|>"
+        user_prompt   = f"<|im_start|>user\n{user_msg}<|im_end|>"
+        assistant_pref= f"<|im_start|>assistant\n<think>"
+        curr_prompt   = system_prompt + "\n" + user_prompt + "\n" + assistant_pref
+
+        # 2️⃣ normal ReCall loop hitting the HF inference endpoint
+        for _ in range(10):
+            resp = requests.post(
+                f"{model_url}/generate",
+                json={
+                    "text": curr_prompt,
+                    "sampling_params": {
+                        "temperature": temperature,
+                        "max_new_tokens": max_new_tokens,
+                    }
+                },
+                timeout=120,
+            ).json()
+            if "error" in resp.keys():
+                print("resp", resp)
+            assistant_txt = resp["text"]
+            curr_prompt = self.cat_assistant_response(curr_prompt, assistant_txt)
+
+            tool_calls = self.extract_tool_calls(assistant_txt)
+            if  len(tool_calls) != 0:
+                # break  # model produced an answer → done
+
+                results = self.execute_tool_calls(env, tool_calls)
+                curr_prompt = self.cat_tool_results(curr_prompt, tool_calls, results)
+
+            else:
+                continue
+        return curr_prompt
+    
+
+
+    @retry(max=5, sleep=1, fallback={"score": 0}) 
+    def run_budget(
+        self,
+        env: str,
+        func_schemas: str,
+        question: str,
+        model_url: str = "http://0.0.0.0:1214",
+        temperature: float = 0.0,
+        max_new_tokens: int = 2048,
+    ) -> str:
+        """
+        Execute an agentic dialogue with external tools while *pruning* previous
+        <tool_response> blocks to prevent context-length explosion.
+        """
+        curr_prompt = self.init_prompt(func_schemas, question)
+
+        for _ in range(16):  # hard loop-limit
+            # ── 1. Call the model 
+            rsp = requests.post(
+                f"{model_url}/generate",
+                json={
+                    "text": curr_prompt,
+                    "sampling_params": {
+                        "temperature": temperature,
+                        "max_new_tokens": max_new_tokens,
+                        "stop": ["<|im_end|>", "</think>", "</think>\n" "</think>\n\n"],
+                    },
+                   
+                },
+                timeout=120,
+            ).json()
+            generated = rsp["text"]                       # what you have now
+            matched   = rsp["meta_info"]["finish_reason"].get("matched")
+
+            # ⇢ append the tag back only if it was removed
+            if matched and not generated.endswith(matched):
+                generated += matched
+
+            # Fail fast on server error
+            if "error" in rsp:
+                raise RuntimeError(rsp["error"])
+
+            assistant_text: str = rsp["text"]
+            curr_prompt = self.cat_assistant_response(curr_prompt, assistant_text)
+
+            # ── 2. Check for final answer ────────────────────────────────────
+            if "<answer>" in assistant_text:
+                break
+
+            # ── 3. Extract & execute tool calls ──────────────────────────────
+            tool_calls: List[str] = self.extract_tool_calls(assistant_text)
+            if not tool_calls:        # continue reasoning without calling a tool
+                continue
+
+            results: List[str] = self.execute_tool_calls(env, tool_calls)
+
+
+            # ── 4. BEFORE appending new tool output, drop all old ones ───────
+            curr_prompt =self. _strip_old_tool_responses(curr_prompt)
+
+            # ── 5. Append *only* the fresh tool_response block ───────────────
+            curr_prompt = self.cat_tool_results(curr_prompt, tool_calls, results)
+
+        return curr_prompt
+
+ 
+
+  
+    def _strip_old_tool_responses_msgs(self, messages: list[dict]) -> list[dict]:
+        """
+        Return a copy of `messages` with every *user* message that starts with
+        <tool_response> removed.  Keeps assistant turns untouched.
+        """
+        return [
+            m for m in messages
+            if not (m["role"] == "user" and m["content"].lstrip().startswith("<tool_response>"))
+        ]
+    # ────────── budget version ──────────
+    @retry(max=5, sleep=1, fallback={"score": 0})
+    def run_deepseek_budget(
+        self,
+        env: str,
+        func_schemas: str,
+        question: str,
+        api_key: str,
+        model_name: str,
+        temperature: float = 0.0,
+        top_p: float = 0.95,
+        max_tokens: int = 32768,
+        max_turns: int = 10,
+    ):
+        """
+        Chat-based ReCall loop for DeepSeek-R1 **with context-budget pruning**.
+        Keeps only the *latest* <tool_response> block to avoid prompt bloat.
+        """
+        sys_content = self.system_prompt_budget.format(func_schemas=func_schemas)
+
+        messages = [
+            {"role": "system", "content": sys_content},
+            {"role": "user",   "content": question},
+        ]
+
+        client = Together(api_key=api_key)
+
+        for turn in range(max_turns):
+            # ── 1. model call ───────────────────────────────────────────────
+            resp = client.chat.completions.create(
+                model=model_name,
+                messages=messages,
+                temperature=temperature,
+                top_p=top_p,
+                max_tokens=max_tokens,
+                stop=["</tool_call>", "<｜end▁of▁sentence｜>"],
+            )
+            assistant_text = resp.choices[0].message.content
+            messages.append({"role": "assistant", "content": assistant_text})
+
+            print(f"**assistant** \n {assistant_text}")
+
+            # ── 2. finished? ────────────────────────────────────────────────
+            if "<answer>" in assistant_text:
+                break
+
+            # ── 3. parse tool calls ────────────────────────────────────────
+            tool_calls = self.extract_tool_calls(assistant_text)
+            print(f"**tool_calls** \n {tool_calls}")
+            if not tool_calls:
+                continue  # keep reasoning without tools
+
+            # ── 4. execute tools ───────────────────────────────────────────
+            results = self.execute_tool_calls(env, tool_calls)
+            print(f"**tool_response** \n {results}")
+
+            # ── 5. prune & append fresh tool_response ──────────────────────
+            messages = self._strip_old_tool_responses_msgs(messages)
+
+            tool_resp_block = "".join(
+                f"<tool_response>{c}\n{r}\n</tool_response>\n"
+                for c, r in zip(tool_calls, results)
+            )
+            messages.append({"role": "user", "content": tool_resp_block})
+
+        # ── 6. flatten & return trajectory (sans system for readability) ───
+        trajectory = "\n".join(
+            f"<{m['role']}>\n{m['content']}" for m in messages if m["role"] != "system"
+        )
+        return trajectory
+
+
+    @retry(max=5, sleep=1, fallback=None)
+    def run_deepseek_with_prompt_injection(
+        self,
+        env: str,
+        func_schemas: str,
+        question: str,
+        api_key: str,
+        model_name: str,
+        temperature: float = 0.0,
+        top_p: float = 0.95,
+        max_tokens: int = 32768,
+    ):
+        """
+        1) Call pubmed_search(question, top_n=5) as a tool to get snippets.
+        2) Inject them into the first user message.
+        3) Proceed with the usual DeepSeek-R1 tool‐based rollout.
+        """
+
+        # ── Step 0: prepare the single‐tool call for retrieval ───────────────
+        retrieve_call = json.dumps({
+            "name": "pubmed_search",
+            "arguments": {
+                "query": question,
+                "top_n": 5
+            }
+        })
+
+        # Execute it once via your helper
+        # note: `env` must include whatever import / client‐setup
+        #          your sandbox needs to run pubmed_search(...)
+        raw_retrieval_results = self.execute_tool_calls(env, [retrieve_call])[0]
+        # print("AAAAA"*100)
+        try:
+            snippets = raw_retrieval_results[9:] #"remove result: str"
+            # print(snippets)
+        except:
+            snippets = ""
+            # print(f"[ReCall] Retriever call failed to parse JSON, got:\n{raw_retrieval_results!r}")
+
+        # ── Step 1: build the injected user prompt ────────────────────────────
+        if snippets:
+            
+            user_content = (
+                f"Question: {question}\n\n"
+                "Here are some relevant PubMed snippets:\n"
+                f"{snippets}"
+            )
+        else:
+            user_content = f"Question: {question}"
+
+        # ── Step 2: start the chat history ────────────────────────────────────
+        sys_content = self.system_prompt_forcing_tool_call
+        messages = [
+            {"role": "system",  "content": sys_content},
+            {"role": "user",    "content": user_content},
+        ]
+        client = Together(api_key=api_key)
+
+        # ── Step 3: your normal ReCall tool‐calling loop ─────────────────────
+        for turn in range(10):
+            resp = client.chat.completions.create(
+                model       = model_name,
+                messages    = messages,
+                temperature = temperature,
+                top_p       = top_p,
+                max_tokens  = max_tokens,
+                stop        = ["</tool_call>", "<｜end▁of▁sentence｜>"]
+            )
+
+            assistant_text = resp.choices[0].message.content
+            messages.append({"role": "assistant", "content": assistant_text})
+
+            tool_calls = self.extract_tool_calls(assistant_text)
+            if not tool_calls:
+                break
+
+            # Execute all of the tool calls in one go
+            results = self.execute_tool_calls(env, tool_calls)
+            # and append them back in the required <tool_response> format
+            tool_resp_block = "".join(
+                f"<tool_response>{call}\n{out}\n</tool_response>\n"
+                for call, out in zip(tool_calls, results)
+            )
+            messages.append({"role": "user", "content": tool_resp_block})
+
+        # ── Step 4: flatten to a single trajectory ────────────────────────────
+        trajectory = "\n".join(
+            f"<{m['role']}>\n{m['content']}"
+            for m in messages
+            if m["role"] != "system"
+        )
+        return trajectory
+

requirements.txt

@@ -0,0 +1,16 @@
+transformers
+accelerate
+openai
+tiktoken 
+pix2tex
+Pillow
+gradio
+fastapi
+pydantic
+uvicorn
+together
+beautifulsoup4
+trafilatura
+wikipedia
+PyMuPDF
+Crawl4AI

run_question.py

@@ -0,0 +1,275 @@
+# single_question_recall.py
+from __future__ import annotations
+import re
+import os
+from typing import Any, Dict, Optional
+
+from re_call import ReCall
+from transformers import AutoTokenizer
+
+
+import re
+from typing import Optional, Any, Dict, Tuple, List
+
+def _extract_answer_boxed(s: str) -> Optional[str]:
+    """
+    Return the content of the *last* \\boxed{...} or \\fbox{...} in `s`,
+    with proper matching of nested braces. Escaped braces (\\{, \\}) are ignored
+    for counting. If no balanced block is found, returns None.
+    """
+    def _iter_box_like_spans(text: str):
+        # Find openings for \boxed{ and \fbox{
+        openings: List[Tuple[str, int, int]] = []
+        for m in re.finditer(r'\\boxed\s*\{', text):
+            openings.append(("boxed", m.start(), m.end()))
+        for m in re.finditer(r'\\fbox\s*\{', text):
+            openings.append(("fbox", m.start(), m.end()))
+        openings.sort(key=lambda x: x[1])
+        # For each opening, scan forward to find its matching closing brace
+        for kind, start, open_end in openings:
+            depth = 1
+            i = open_end
+            n = len(text)
+            while i < n:
+                ch = text[i]
+                # Skip escaped characters: backslash escapes the next char (including { or })
+                if ch == '\\' and i + 1 < n:
+                    i += 2
+                    continue
+                if ch == '{':
+                    depth += 1
+                elif ch == '}':
+                    depth -= 1
+                    if depth == 0:
+                        # content is text[open_end:i]
+                        yield (kind, start, open_end, i)
+                        break
+                i += 1
+
+    last_content: Optional[str] = None
+    for _, _start, open_end, close_idx in _iter_box_like_spans(s):
+        last_content = s[open_end:close_idx]  # keep the *last* one
+
+    return last_content.strip() if last_content is not None else None
+
+
+def _extract_answer_tagged(s: str) -> Optional[str]:
+    answer_tag_re = re.compile(r"<answer>(.*?)</answer>", re.S)
+    m = answer_tag_re.findall(s)
+    return m[-1].strip() if m else None
+
+
+def _parse_answer_from_transcript(transcript: str) -> str:
+    """
+    Prefer balanced \\boxed{...}/\\fbox{...} content, then <answer>...</answer>,
+    else fall back to the last 200 chars.
+    """
+    return (
+        _extract_answer_boxed(transcript)
+        or _extract_answer_tagged(transcript)
+        # or transcript[-200:].strip()
+    )
+
+
+# --- main API: recall only ---
+def answer_question_recall(
+    question: str,
+    *,
+    model_url: Optional[str] = None,         # your thinker endpoint (if recall uses one)
+    executor_url: Optional[str] = None,
+    tokenizer_dir: str = "./tokenizer-info",
+    temperature: float = 0.6,
+    max_new_tokens: int = 40960,
+    top_p: float = 0.95,
+    search_env: str = "from search_api import search_urls, open_url, search_and_parse_query, query_url",
+    func_schemas = [
+        {
+            "name": "search_urls",
+            "description": "Google search and return links to web-pages with a brief snippet given a text query",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}, "top_k": {"type": "integer", "default": 10}},
+                "required": ["query"],
+            },
+        },
+        {
+            "name": "query_url",
+            "description": "Visit webpage and return evidence based retrival for the provided goal",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "url": {"type": "string", "description": "The URL of the webpage to visit. Must be a single URL"},
+                    "goal": {"type": "string", "description": "The specific information goal for visiting webpage"},
+                },
+                "required": ["url", "goal"],
+            },
+        },
+    ],
+    deepseek_name: str = "deepseek-ai/DeepSeek-R1",
+    old_prompt: Optional[str] = None,
+    deepresearch_on: bool = True,
+    summary_llm: str = "gpt-4.1-mini",
+    ):
+    # ) -> Dict[str, Any]:
+    """
+    Runs a single question through ReCall and returns:
+    {
+      "answer": str,
+      "transcript": str,
+      "tool_calls": Any,
+      "chat": Any | None
+    }
+    """
+    if executor_url is None:
+        executor_url = os.environ["HOST_SERPER_URL"]
+        
+    if model_url is None:
+        model_url = os.environ["HF_MODEL_URL"]
+
+    # 1) tokenizer (REQUIRED by ReCall.run)
+    tok = AutoTokenizer.from_pretrained(tokenizer_dir, trust_remote_code=True)
+
+    # 2) build agent
+    agent = ReCall(executor_url=executor_url)
+    
+    last_out = ""
+
+    # 3) call the correct entrypoint
+    if model_url == deepseek_name:
+        # some setups use a special deepseek path that returns (transcript, tool_calls)
+        out = agent.run_deepseek(
+            env=search_env,
+            func_schemas=func_schemas,
+            question=question,
+            model_name=deepseek_name,
+            temperature=temperature,
+            max_tokens=max_new_tokens,
+            top_p=top_p,
+        )
+        transcript, tool_calls, chat = _normalize_out(out, expect_chat=False)
+        last_out = transcript
+    else:
+        # standard ReCall.run MUST receive tokenizer
+        agent_generator = agent.run(
+            env=search_env,
+            func_schemas=func_schemas,
+            question=question,
+            model_url=model_url,
+            temperature=temperature,
+            max_new_tokens=max_new_tokens,
+            tokenizer=tok, # <- fixes your "missing tokenizer" error
+            top_p=top_p,
+            old_prompt=old_prompt,# <- you can pass the raw old prompt here if there exists an older chat history
+            # the function will append the question to the raw old prompt string (chat history) if it is not None
+            deepresearch_on=deepresearch_on,
+            summary_llm=summary_llm
+            # deepresearch=deepresearch, # <- use the deepresearch prompt
+        )
+        
+        while True:
+            try:
+                tag, out = next(agent_generator)
+                if tag == "assistant_resp":
+                    last_out = out[0]
+                yield tag, out
+                if tag == "end":
+                    break
+            except StopIteration as e:
+                # the chat_str variable contains the whole conversation in the raw string form
+                # it contains the raw tokens like "<|im_start|>system\n", "<|im_end|>"
+                # and "<|im_start|>assistant\n<think>", "<tool_response>", "\n</tool_response>\n", etc.
+                chat_str: str = e.value[1][0]
+                yield "end", (chat_str,)
+                break
+
+    # 4) parse final answer
+    answer = _parse_answer_from_transcript(last_out)
+
+    return "answer", (answer,)
+
+
+def _normalize_out(out: Any, expect_chat: bool) -> tuple[str, Any, Any]:
+    """
+    Normalize ReCall outputs to (transcript, tool_calls, chat)
+    Handles:
+      - (transcript, tool_calls, chat)
+      - (transcript, tool_calls)
+      - "transcript"
+      - {"transcript": ..., "tool_calls": ..., "chat": ...} variants
+    """
+    transcript, tool_calls, chat = "", None, None
+
+    if isinstance(out, tuple):
+        if len(out) == 3:
+            transcript, tool_calls, chat = out
+        elif len(out) == 2:
+            transcript, tool_calls = out
+        elif len(out) == 1:
+            transcript = out[0]
+        else:
+            transcript = str(out[-1])
+    elif isinstance(out, dict):
+        transcript = out.get("transcript") or out.get("output") or out.get("response") or ""
+        tool_calls = out.get("tool_calls")
+        chat = out.get("chat")
+    else:
+        transcript = str(out)
+
+    # Some implementations return None/empty; keep things predictable
+    if chat is None and expect_chat is False:
+        chat = None
+    return transcript, tool_calls, chat
+
+
+# quick demo
+if __name__ == "__main__":
+    old_prompt = None
+    
+    answer_generator = answer_question_recall(
+        "What is the most popular restraunt in kolkata?",
+        old_prompt=old_prompt
+    )
+    
+    # print("ANSWER:", res["answer"])
+    # print("\n")
+    # # print(type(res["tool_calls"]), len(res["tool_calls"]))
+    # for i in res["tool_calls"]:
+    #     print(f"{i}\n")
+    # print("\n")
+    # if res["chat"] is not None:
+    #     # print(type(res["chat"]), len(res["chat"]))
+    #     for i in res["chat"]:
+    #         print(f"{i}\n")
+    #     print("\n")
+    # print("TRANSCRIPT (tail):\n", res["transcript"][-300:])
+    
+    final_chat_str = ""
+    
+    while True:
+        try:
+            tag, out = next(answer_generator)
+            if tag == "assistant_resp":
+                assistant_text, tool_calls = out
+                print(f"ASSISTANT RESPONSE:\n{assistant_text}\n\n")
+                print("TOOL CALLS:\n")
+                for tool_call in tool_calls:
+                    print(f"{tool_call}")
+                print("\n")
+            elif tag == "tool_results":
+                results = out[0]
+                print("TOOL RESULTS:\n")
+                for result in results:
+                    print(f"{result}")
+                print("\n")
+            elif tag == "end":
+                print(f"{'='*20}\nASSISTANT RESPONSE ENDED\n{'='*20}\n\n")
+                final_chat_str = out[0]
+            elif tag == "answer":
+                answer = out[0]
+                print(f"FINAL ANSWER:\n{answer}\n\n")
+                break
+        except StopIteration as e:
+            print(f"FINAL ANSWER:\n{e.value[1][0]}\n\n")
+            break
+    
+    print(f"{'='*20}\nEND\n{'='*20}\n\n\nFINAL CHAT STRING:\n{final_chat_str}\n\n")

tokenizer-info/added_tokens.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0284b582e14987fbd3d5a2cb2bd139084371ed9acbae488829a1c900833c680
+size 707

tokenizer-info/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76862e765266b85aa9459767e33cbaf13970f327a0e88d1c65846c2ddd3a1ecd
+size 613

tokenizer-info/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

tokenizer-info/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:443bfa629eb16387a12edbf92a76f6a6f10b2af3b53d87ba1550adfcf45f7fa0
+size 5404

tokenizer-info/vocab.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910
+size 2776833

web_agents_5/compressor.py

@@ -0,0 +1,314 @@
+# compressor.py
+from __future__ import annotations
+import functools, json, logging, re
+from difflib import SequenceMatcher
+from io import StringIO
+from typing import Dict, List, Tuple
+
+import pandas as pd
+import regex  # needed by tiktoken
+import tiktoken
+from bs4 import BeautifulSoup
+from config import CFG
+from web_helpers import retry
+
+# ────────────────────────────────────────────────────────────────────────
+# 0. shared helpers
+# ------------------------------------------------------------------------
+enc = tiktoken.get_encoding("cl100k_base")
+_tok = lambda s: len(enc.encode(s))                     # fast inline counter
+
+@functools.lru_cache(maxsize=1)
+def _nlp():
+    import spacy
+    return spacy.load("en_core_web_sm")
+
+def _openai_client():
+    """Import OpenAI lazily to avoid overhead when not needed."""
+    import importlib
+    mod = importlib.import_module("openai")
+    return getattr(mod, "OpenAI", None)() if hasattr(mod, "OpenAI") else mod
+
+# ────────────────────────────────────────────────────────────────────────
+# 1. regex patterns (compiled once)
+# ------------------------------------------------------------------------
+DATE_PATS   = [re.compile(p, re.I) for p in [
+    r"\d{4}-\d{2}-\d{2}",
+    r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},\s+\d{4}",
+    r"(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},\s+\d{4}",
+    r"\d{1,2}\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{4}",
+    r"\b\d{4}/\d{2}\b",
+    r"\b\d{4}\b(?!\s*(?:%|million|billion|thousand))",
+]]
+EMAIL_PAT   = re.compile(r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+")
+URL_PAT     = re.compile(r"https?://[^\s\)]+")
+PHONE_PAT   = re.compile(r"\+?\d[\d\s\-().]{7,}\d")
+CURR_PAT    = re.compile(r"(\$\s?\d+(?:,\d{3})*(?:\.\d+)?|\d+(?:,\d{3})*(?:\.\d+)?\s*(USD|EUR|GBP|INR|¥|₩|₹|€))", re.I)
+DEF_PAT     = re.compile(r"([A-Z][A-Za-z0-9\s]+?)\s+(is|are|refers to|means)\s+(.*?)(?:[\.\n])")
+
+MD_TABLE_PAT = re.compile(
+    r"(?:^\|.*?\|\n?)+(?:^\|[-:\s|]+\|\n?)?(?:^\|.*?\|\n?)+", re.M)
+CSV_PAT      = re.compile(r"((?:^.*?,.*?\n){2,})", re.M)
+TSV_PAT      = re.compile(r"((?:^.*?\t.*?\n){2,})", re.M)
+
+# ────────────────────────────────────────────────────────────────────────
+# 2. core utilities
+# ------------------------------------------------------------------------
+def deduplicate_items(items: List[str], *, similarity=0.5,
+                      other: List[str] | None = None) -> List[str]:
+    """Drop near‑duplicates; prefer the longest variant."""
+    if not items: return []
+    other = other or []
+
+    def _clean(x: str) -> str:
+        x = re.sub(r'\[edit\]|\[\d+\]', '', x)
+        return re.sub(r'\s+', ' ', x).strip()
+
+    out, out_clean = [], []
+    for orig in items:
+        clean = _clean(orig)
+        dup = False
+        for ref in out_clean + list(map(_clean, other)):
+            sim = SequenceMatcher(None, clean, ref).ratio()
+            if sim >= similarity or clean in ref or ref in clean:
+                dup = True
+                # if current is longer than stored, replace
+                if clean not in out_clean and len(clean) > len(ref):
+                    idx = out_clean.index(ref)
+                    out[idx], out_clean[idx] = orig, clean
+                break
+        if not dup:
+            out.append(orig)
+            out_clean.append(clean)
+    return out
+
+# ────────────────────────────────────────────────────────────────────────
+# 3. fact & table extractor
+# ------------------------------------------------------------------------
+def extract_facts_and_tables(text: str) -> Tuple[str, List[str], List[str]]:
+    facts, spans = [], []
+
+    def _add(match):
+        facts.append(match.group())
+        spans.append(match.span())
+
+    for pat in DATE_PATS:   [_add(m) for m in pat.finditer(text)]
+    for m in EMAIL_PAT.finditer(text):   _add(m)
+    for m in URL_PAT.finditer(text):     _add(m)
+    for m in PHONE_PAT.finditer(text):   _add(m)
+    for m in CURR_PAT.finditer(text):    _add(m)
+    for m in DEF_PAT.finditer(text):     _add(m)
+
+    # contextual sentences around facts
+    doc = _nlp()(text)
+    ctx = [s.text.strip() for s in doc.sents
+           if any(s.start_char <= s_ <= s.end_char for s_, _ in spans)]
+    facts.extend(ctx)
+    facts = sorted(set(facts))
+
+    # ── tables
+    tables = []
+
+    for tbl in MD_TABLE_PAT.findall(text):
+        cleaned = "\n".join(l for l in tbl.splitlines()
+                            if l.strip() and not re.match(r"^\|[-:\s|]+\|$", l))
+        if len(cleaned.splitlines()) < 2: continue
+        try:
+            df = pd.read_csv(StringIO(cleaned), sep="|").dropna(how="all", axis=1)
+            tables.append(df.to_markdown(index=False))
+        except Exception:
+            tables.append(cleaned)
+
+    soup = BeautifulSoup(text, "lxml")
+    for html_tbl in soup.find_all("table"):
+        try:
+            df = pd.read_html(str(html_tbl))[0]
+            tables.append(df.to_markdown(index=False))
+        except Exception:
+            tables.append(str(html_tbl))
+
+    for m in CSV_PAT.finditer(text):
+        try:
+            df = pd.read_csv(StringIO(m.group(1)))
+            if not df.empty:
+                tables.append(df.to_markdown(index=False))
+        except Exception:
+            pass
+    for m in TSV_PAT.finditer(text):
+        try:
+            df = pd.read_csv(StringIO(m.group(1)), sep="\t")
+            if not df.empty:
+                tables.append(df.to_markdown(index=False))
+        except Exception:
+            pass
+
+    # ── clean narrative (remove facts & tables)
+    narrative = text
+    for tbl in tables: narrative = narrative.replace(tbl, " ")
+    for s, e in sorted(spans, reverse=True): narrative = narrative[:s] + narrative[e:]
+    narrative = re.sub(r"\s{2,}", " ", narrative).strip()
+
+    return narrative, facts, tables
+
+# ────────────────────────────────────────────────────────────────────────
+# 4. OpenAI summariser helpers
+# ------------------------------------------------------------------------
+def _summarise(text: str, pct: float, model: str) -> str:
+    target_tokens = int(_tok(text) * pct)
+    sys_prompt = (
+        "You are an expert abstractor. Summarize the text below to "
+        f"approximately {pct*100:.0f}% of its original length (≈{target_tokens} tokens), "
+        "while **retaining all key facts, figures, names, dates, places, and events**. "
+        "Ensure the summary remains accurate, informative, and faithful to the original content."
+    )
+    client = _openai_client()
+    rsp = client.chat.completions.create(
+        model=model, temperature=0.2,
+        messages=[{"role":"system","content":sys_prompt},
+                  {"role":"user","content":text}],
+        max_tokens=CFG.output_limit_per_link
+    )
+    return rsp.choices[0].message.content
+
+# ────────────────────────────────────────────────────────────────────────
+# 5. compress_text  (public)
+# ------------------------------------------------------------------------
+def compress_text(text: str, *, pct: float = 0.3,
+                  model: str = "gpt-4o-mini") -> Dict[str, str]:
+
+    FACTS_TABLES_LIMIT = CFG.output_limit_per_link - CFG.disable_narrative_compress_thresh
+    narrative, facts, tables = extract_facts_and_tables(text)
+
+    # narrative compression
+    if _tok(narrative) > CFG.disable_narrative_compress_thresh:
+        narrative_txt = _summarise(narrative, pct, model)
+    else:
+        narrative_txt = narrative
+    return narrative_txt
+
+# ────────────────────────────────────────────────────────────────────────
+# 6. query_text  (goal‑oriented extraction)
+# ------------------------------------------------------------------------
+EXTRACTOR_SYS_PROMPT = (
+    "You are a highly skilled information extraction agent. Your job is to analyze long, complex webpages "
+    "in the context of a specific user goal. You excel at identifying relevant sections, capturing supporting evidence "
+    "in full original context, and providing logically structured summaries. Always ensure precision, completeness, "
+    "and alignment with the user’s intent."
+)
+EXTRACTOR_PROMPT_TEMPLATE = """You are a highly skilled information extraction agent. Your task is to analyze the following webpage content in light of a specific user goal, and extract accurate, well-structured information using plain text format.
+
+## Webpage Content
+{webpage_content}
+
+## User Goal
+{goal}
+
+## Task Guidelines
+1. **Rational**: Briefly explain why this content is relevant to the user’s goal.
+2. **Evidence**: Quote the most relevant parts of the webpage that directly support or address the goal. Use bullet points or numbered lines separated by newlines.
+3. **Summary**: Provide a clear, logically structured summary of the extracted evidence that addresses the user's goal.
+
+## Output Format
+Your response must follow **exactly this format** with the three sections:
+Rational: <one paragraph>
+Evidence: <first point>\n<second point>...
+Summary:<concise paragraph summarizing the evidence>
+"""
+
+def extract_regex(text: str) -> Dict[str, str]:
+    def extract_section(header: str) -> str:
+        # Match the section starting with `Header:` until the next capitalized line followed by `:` or end
+        pattern = rf"{header}:\s*(.*?)(?=\n[A-Z][a-z]+:|\Z)"
+        match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
+        return match.group(1).strip() if match else "(not found)"
+
+    return {
+        "rational": extract_section("Rational"),
+        "evidence": extract_section("Evidence"),
+        "summary": extract_section("Summary")
+    }
+
+def query_text(
+    url: str,
+    text: str,
+    goal: str,
+    *,
+    model: str = "gpt-4.1-mini",
+    max_attempts: int = 3,
+) -> Dict[str, str]:
+    """Goal‑oriented extractor with retries → compress fallback → token trim fallback."""
+    prompt = EXTRACTOR_PROMPT_TEMPLATE.format(
+        webpage_content=text[:15_000],  # clip for safety
+        goal=goal,
+    )
+    client = _openai_client()
+
+    for attempt in range(1, max_attempts + 1):
+        try:
+            rsp = client.chat.completions.create(
+                model=model,
+                temperature=0.0,
+                messages=[
+                    {"role": "system", "content": EXTRACTOR_SYS_PROMPT},
+                    {"role": "user", "content": prompt},
+                ],
+                max_tokens = 1024
+            ).choices[0].message.content
+
+            extracted = extract_regex(rsp)
+
+            # Sanity check: evidence + summary must be > 20 characters
+            if len(extracted.get("evidence", "")) + len(extracted.get("summary", "")) > 20:
+                return {
+                    "extracted_info": (
+                        f"The useful information in {url} for goal “{goal}”:\n\n"
+                        f"Rationale:\n{extracted.get('rational')}\n\n"
+                        f"Evidence:\n{extracted.get('evidence')}\n\n"
+                        f"Summary:\n{extracted.get('summary')}"
+                    )
+                }
+
+            raise ValueError("LLM returned empty or malformed extraction")
+
+        except Exception as e:
+            logging.warning("Attempt %d/%d failed for query-based extraction: %s",
+                            attempt, max_attempts, e)
+
+    # ── Retry fallback: compress text ─────────────────────────────────────
+    try:
+        compressed = compress_text(text, model=model)
+        return {
+            "extracted_info": (
+                f"Goal-based extraction failed after {max_attempts} attempts; "
+                f"returning compressed webpage:\n\n{compressed}"
+            )
+        }
+    except Exception as ce:
+        logging.error("compress_text also failed: %s", ce)
+
+    # ── Final fallback: hard truncate to token budget ────────────────────
+    return {
+        "extracted_info": (
+            "Goal-based extraction and compression both failed; "
+            "returning truncated webpage content:\n\n" +
+            trim_to_budget(text, CFG.output_limit_per_link, model=model)
+        )
+    }
+
+   
+# ────────────────────────────────────────────────────────────────────────
+# 7. helper: trim long lists to token budget
+# ------------------------------------------------------------------------
+def trim_to_budget(items: List[str], budget: int, *,
+                    is_table: bool) -> Tuple[str, int]:
+    build, used = [], 0
+    for it in items:
+        toks = _tok(it)
+        if used + toks > budget:
+            break
+        build.append(it)
+        used += toks
+    if len(build) < len(items):
+        build.append(f"[{len(items)-len(build)} {'tables' if is_table else 'facts'} omitted]")
+    joined = "\n\n".join(build) if is_table else "\n".join(build)
+    return joined, _tok(joined)

web_agents_5/config.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+import logging, os, random, requests
+
+class _Cfg:
+    ua: str = (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/125.0 Safari/537.36"
+    )
+    serper_key            = os.getenv("SERPER_API_KEY", "")
+    jina_cache_dir             =  os.getenv("JINA_CACHE_DIR", "")
+    serper_cache_dir             =  os.getenv("SERPER_CACHE_DIR", "")
+    jina_key             = os.getenv("JINA_API_KEY", "")
+    serper_ep             = "https://google.serper.dev/search"
+    crawl4ai_ep           = os.getenv("CRAWL4AI_EP", "http://localhost:8080")
+    retries               = 3
+    backoff               = 0.8
+    connect_to            = 5
+    read_to               = 10
+    stream_html_cap       = 200_000
+    pdf_size_cap          = 32_000_000
+    pdf_pages_cap         = 40
+    pdf_chars_cap         = 40_000
+    text_cap              = 400_000
+    output_limit_per_link = 6_000
+    disable_narrative_compress_thresh = 2_000
+    pct                  = 0.25          # narrative compression pct
+    reddit_client_id = "Q2tovcGfYmo3hPNvzTpkXA"
+    reddit_client_secret = "geu4gH3pEOrNnsMpQvdTTVhQvDABgg"
+
+
+CFG   = _Cfg()
+_RND  = random.Random()
+_SESS = requests.Session()
+_SESS.headers.update({"User-Agent": CFG.ua})
+
+# logging.basicConfig(level=logging.INFO,
+#                     format="%(asctime)s - %(levelname)s - %(message)s")
+logging.getLogger().setLevel(logging.INFO)   # bump root to DEBUG

web_agents_5/fetchers/basic_fetcher.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+import logging
+from urllib.parse import unquote
+from config import CFG, _SESS
+from web_helpers import extract_main_text, fetch_blocked_site
+
+_BINARY = (".jpg", ".jpeg", ".png", ".gif", ".bmp", ".zip", ".tar",
+           ".gz", ".mp3", ".mp4", ".mkv", ".exe")
+
+_ERROR = ["wrong", "error", "try again"]
+
+def _looks_like_error(txt):
+    if len(txt) < 300:
+        for err in _ERROR:
+            if err in txt:
+                return True
+    return False 
+
+
+def fetch_html(url: str) -> str:
+    if url.lower().endswith(_BINARY):
+        return "[binary omitted]"
+    try:
+        r = _SESS.get(url, stream=True, timeout=(CFG.connect_to, CFG.read_to))
+        r.raise_for_status()
+        ctype = (r.headers.get("content-type") or "").lower()
+        if "pdf" in ctype or not ("text" in ctype or "html" in ctype):
+            return "[binary omitted]"
+        raw = r.raw.read(CFG.stream_html_cap, decode_content=True)
+        html = raw.decode(r.encoding or "utf-8", errors="ignore")
+        txt  = extract_main_text(html).strip()
+        if "wikipedia.org" in url:
+            slug = unquote(url.rsplit("/", 1)[-1]).replace("_", " ")
+            if slug.lower() not in txt.lower():
+                txt = f"{slug}\n\n{txt}"
+        if _looks_like_error(txt):
+            return f"[Error fetching url: {url}]"
+        else:
+            return "[Retrived using HTML] " + txt
+    except Exception as e:
+        logging.error("Generic fetch failed %s: %s", url, e)
+        return fetch_blocked_site(url)

web_agents_5/fetchers/crawl4ai_fetcher.py

@@ -0,0 +1,104 @@
+"""
+Asynchronous wrapper around **Crawl4AI** so that other coroutines can await a
+single call – identical to the previous implementation but isolated in its own
+module to satisfy clean‑architecture / layering.
+
+Public API
+==========
+async def fetch_crawl4ai(url: str) -> str
+    Returns markdown extracted by Crawl4AI or raises `RuntimeError` on failure.
+"""
+from __future__ import annotations
+
+import asyncio, logging
+from dataclasses import dataclass, field
+from typing import Any
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from config import CFG
+
+# ----------------------------------------------------------------------------
+_MAX_CONCURRENT_PAGES = 6
+_MAX_ATTEMPTS = 5
+_RETRYABLE = (
+    "handler is closed",
+    "browser has disconnected",
+    "transport closed",
+    "target crashed",
+)
+
+# Globals bound to the *event‑loop* currently active
+_CRAWLER: AsyncWebCrawler | None = None
+_CRAWLER_LOOP: asyncio.AbstractEventLoop | None = None
+_SEMAPHORES: dict[asyncio.AbstractEventLoop, asyncio.Semaphore] = {}
+_CFG = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator())
+
+
+def _get_semaphore() -> asyncio.Semaphore:
+    loop = asyncio.get_running_loop()
+    if loop not in _SEMAPHORES:
+        _SEMAPHORES[loop] = asyncio.Semaphore(_MAX_CONCURRENT_PAGES)
+    return _SEMAPHORES[loop]
+
+
+async def _ensure_crawler() -> None:
+    global _CRAWLER, _CRAWLER_LOOP
+    loop = asyncio.get_running_loop()
+    if _CRAWLER is None or loop is not _CRAWLER_LOOP:
+        if _CRAWLER is not None:
+            try:
+                await _CRAWLER.aclose()
+            except Exception:
+                pass
+        _CRAWLER = AsyncWebCrawler()
+        await _CRAWLER.__aenter__()
+        _CRAWLER_LOOP = loop
+
+
+@dataclass
+class _Page:
+    success: bool
+    markdown: str | None = None
+    error: str | None = None
+    meta: dict[str, Any] = field(default_factory=dict)
+
+
+async def _crawl_once(url: str) -> _Page:
+    global _CRAWLER
+    await _ensure_crawler()
+
+    try:
+        result = await _CRAWLER.arun(url, config=_CFG)
+        if result.success and result.markdown:
+            return _Page(True, result.markdown, meta=result.__dict__)
+        return _Page(False, error=result.error_message or "no markdown")
+    except Exception as exc:
+        return _Page(False, error=str(exc))
+
+
+async def fetch_crawl4ai(url: str) -> str:
+    """Return markdown extracted by Crawl4AI or raise on failure."""
+    sem = _get_semaphore()
+    async with sem:
+        for attempt in range(1, _MAX_ATTEMPTS + 1):
+            page = await _crawl_once(url)
+            if page.success and page.markdown:
+                print(len(page.markdown))
+                return "[Retrieved from Craw4AI]" + page.markdown[:CFG.text_cap]
+
+            err = page.error or "unknown"
+            logging.warning("Crawl4AI attempt %d/%d failed: %s", attempt, _MAX_ATTEMPTS, err)
+
+            if attempt < _MAX_ATTEMPTS and any(p in err.lower() for p in _RETRYABLE):
+                # reset shared browser & retry after back‑off
+                global _CRAWLER
+                try:
+                    await _CRAWLER.aclose()
+                except Exception:
+                    pass
+                _CRAWLER = None
+                await asyncio.sleep(1.5 * attempt)
+                continue
+
+            raise RuntimeError(err)

web_agents_5/fetchers/github_fetcher.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+from config import CFG, _SESS, _RND
+import logging
+import re
+from bs4 import BeautifulSoup
+import functools
+import random
+import requests
+import trafilatura
+import time 
+from web_helpers import retry, fetch_blocked_site     # ⬅️ shared
+
+
+def fetch_github(url: str) -> str:
+    def _markdown_cleanup(md: str) -> str:
+        md = re.sub(r"```.*?```", "", md, flags=re.S)
+        md = re.sub(r"^#+\s*", "", md, flags=re.M)
+        return re.sub(r"[ \t]{2,}", " ", md).strip()
+
+    hdr = {"User-Agent": "ii-research-bot/0.6"}
+    try:
+        m = re.match(r"https?://github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$", url)
+        if m:
+            owner, repo = m.groups()
+            api = f"https://api.github.com/repos/{owner}/{repo}/readme"
+            hdr_api = hdr | {"Accept": "application/vnd.github.v3.raw"}
+            if (tok := os.getenv("GITHUB_TOKEN")):
+                hdr_api["Authorization"] = f"Bearer {tok}"
+            r = _SESS.get(api, headers=hdr_api, timeout=(CFG.connect_to, CFG.read_to))
+            if r.ok and len(r.text) > 30:
+                return _markdown_cleanup(r.text)[:CFG.text_cap]
+        
+        if "/blob/" in url or "/tree/" in url:
+            raw = re.sub(
+                r"https://github\.com/([^/]+)/([^/]+)/(?:blob|tree)/",
+                r"https://raw.githubusercontent.com/\\1/\\2/",
+                url,
+                count=1,
+            ).split("?", 1)[0]
+            r = _SESS.get(raw, headers=hdr, timeout=(CFG.connect_to, CFG.read_to))
+            if r.ok and "text" in (r.headers.get("content-type") or "") and len(r.text) > 0:
+                return r.text[:CFG.text_cap]
+            
+            raw1 = url + ("?raw=1" if "?" not in url else "&raw=1")
+            r = _SESS.get(raw1, headers=hdr, timeout=(CFG.connect_to, CFG.read_to))
+            if r.ok and "text" in (r.headers.get("content-type") or "") and len(r.text) > 0:
+                return r.text[:CFG.text_cap]
+            
+            plain = url + ("?plain=1" if "?" not in url else "&plain=1")
+            html = _SESS.get(plain, headers=hdr, timeout=(CFG.connect_to, CFG.read_to)).text
+            soup = BeautifulSoup(html, "lxml")
+            pre = soup.find("pre")
+            if pre and len(pre.text) > 10:
+                return pre.text[:CFG.text_cap]
+        
+        if "raw.githubusercontent.com" in url:
+            r = _SESS.get(url.split("?", 1)[0], headers=hdr, timeout=(CFG.connect_to, CFG.read_to))
+            if r.ok and "text" in (r.headers.get("content-type") or ""):
+                return "[Retrieved from raw.githubusercontent.com]" + r.text[:CFG.text_cap]
+        
+        raise RuntimeError("github: unable to retrieve plain text")
+    except Exception as e:
+        logging.error(f"GitHub fetch failed for {url}: {e}")
+        return _fetch_blocked_site(url)

web_agents_5/fetchers/jina_fetcher.py

@@ -0,0 +1,151 @@
+# """
+# Jina AI powered web-page fetcher.
+
+# Provides `fetch_jina(url: str) -> str` which returns a **plain-text or markdown** body
+# prefixed with `[Retrieved from Jina AI]` so callers can recognise the source.
+# If the Jina endpoint cannot return usable text (HTTP error, short / empty body, etc.)
+# this function raises an Exception – letting the orchestrator fall back to other
+# fetchers.
+
+# The implementation is **stateless** and thread-safe – no global mutable state is
+# kept apart from the shared requests session from `config` (mirroring the rest of
+# the code-base).
+# """
+
+# from __future__ import annotations
+
+# import logging
+# import os
+# import urllib.parse as _u
+
+# from config import CFG, _SESS  # shared requests session and config
+# from web_helpers import retry
+
+# _JINA_ENDPOINT = "https://r.jina.ai/{url}"  # Note: will prepend http:// when formatting
+
+
+# @retry
+# def fetch_jina(url: str) -> str:
+#     """Return article text extracted by **Jina AI Read API**.
+
+#     Raises:
+#         RuntimeError – if the endpoint does not yield usable text
+#     """
+#     api_url = _JINA_ENDPOINT.format(url=url)
+#     headers = {
+#         "Authorization": f"Bearer {CFG.jina_key}"
+#     }
+#     logging.debug("Jina fetch → %s", api_url)
+ 
+#     # Make request
+#     r = _SESS.get(api_url, headers=headers, timeout=(CFG.connect_to, CFG.read_to))
+#     r.raise_for_status()
+
+#     txt = r.text.strip()
+
+#     # Treat short or errorful body as failure
+#     if len(txt) < 200 and any(err in txt.lower() for err in ["403", "forbidden", "error"]):
+#         raise RuntimeError("Jina AI returned no content")
+
+#     return "[Retrieved from Jina AI] " + txt[: CFG.text_cap]
+
+"""
+Jina AI powered web-page fetcher with URL-based disk cache.
+
+- Cache key: canonicalized URL (sha256)
+- Cache location: <CFG.cache_dir or $CACHE_DIR or ".cache">/jina_read/
+- Always stores the *raw* Jina body (without the "[Retrieved...]" prefix).
+- Atomic writes via os.replace for basic thread/process safety.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+import urllib.parse as _u
+from typing import Tuple
+
+from config import CFG, _SESS  # shared requests session and config
+from web_helpers import retry
+
+_JINA_ENDPOINT = "https://r.jina.ai/{url}"  # expects a fully-qualified, url-encoded target
+
+
+def _canonicalize_url(url: str) -> str:
+    """Ensure URL has a scheme and is normalized for caching/API calls."""
+    p = _u.urlparse(url.strip())
+    if not p.scheme:
+        # Default to http if missing; Jina reader prefers explicit scheme.
+        p = _u.urlparse("http://" + url.strip())
+
+    # Normalize: lowercase scheme/netloc, drop fragment, keep query & path
+    p = p._replace(scheme=p.scheme.lower(), netloc=p.netloc.lower(), fragment="")
+    # Ensure path is at least "/"
+    path = p.path if p.path else "/"
+    return _u.urlunparse((p.scheme, p.netloc, path, "", p.query, ""))
+
+
+def _cache_paths(nurl: str) -> Tuple[str, str]:
+    """Return (cache_dir, cache_file_path) for a normalized URL."""
+    cache_root = CFG.jina_cache_dir
+    cache_dir = os.path.join(cache_root, "jina_read")
+    os.makedirs(cache_dir, exist_ok=True)
+    h = hashlib.sha256(nurl.encode("utf-8")).hexdigest()
+    return cache_dir, os.path.join(cache_dir, f"{h}.txt")
+
+
+def _load_from_cache(cpath: str) -> str | None:
+    try:
+        if os.path.exists(cpath) and os.path.getsize(cpath) > 0:
+            with open(cpath, "r", encoding="utf-8") as f:
+                return f.read()
+    except Exception as e:
+        logging.debug("Jina cache read failed (%s): %s", cpath, e)
+    return None
+
+
+def _save_to_cache(cpath: str, body: str) -> None:
+    try:
+        tmp = f"{cpath}.tmp.{os.getpid()}"
+        with open(tmp, "w", encoding="utf-8") as f:
+            f.write(body)
+        os.replace(tmp, cpath)  # atomic on the same filesystem
+    except Exception as e:
+        logging.debug("Jina cache write failed (%s): %s", cpath, e)
+
+
+@retry
+def fetch_jina(url: str) -> str:
+    """Return article text extracted by **Jina AI Read API** with disk cache.
+
+    Raises:
+        RuntimeError – if the endpoint does not yield usable text
+    """
+    nurl = _canonicalize_url(url)
+    cache_dir, cpath = _cache_paths(nurl)
+
+    # 1) Try cache
+    cached = _load_from_cache(cpath)
+    if cached:
+        logging.info("Jina fetch (cache hit) ← %s", nurl)
+        return "[Retrieved from Jina AI] " + cached[: CFG.text_cap]
+
+    # 2) Fetch from Jina
+    api_url = _JINA_ENDPOINT.format(url=_u.quote(nurl, safe=""))
+    headers = {"Authorization": f"Bearer {CFG.jina_key}"}
+    logging.debug("Jina fetch (cache miss) → %s", api_url)
+
+    r = _SESS.get(api_url, headers=headers, timeout=(CFG.connect_to, CFG.read_to))
+    r.raise_for_status()
+    body = r.text.strip()
+
+    # 3) Validate
+    if len(body) < 200 and any(err in body.lower() for err in ("403", "forbidden", "error")):
+        raise RuntimeError("Jina AI returned no content")
+
+    # 4) Save to cache (store the raw body; callers always get the standard prefix)
+    _save_to_cache(cpath, body)
+
+    return "[Retrieved from Jina AI] " + body[: CFG.text_cap]
+

web_agents_5/fetchers/pdf_fetcher.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+from config import CFG, _SESS
+import io, logging, re, pymupdf as fitz
+
+from web_helpers import retry, fetch_blocked_site     # ⬅️ shared
+# ----------------------------------------------------------------------
+
+class PDFExtractError(RuntimeError): ...
+
+@retry
+def _download_pdf(url: str) -> bytes:
+    with _SESS.get(url, stream=True, timeout=(CFG.connect_to, CFG.read_to)) as r:
+        r.raise_for_status()
+        total = int(r.headers.get("content-length", 0) or 0)
+        if 0 < total > CFG.pdf_size_cap:
+            raise RuntimeError("pdf too large")
+        buf = io.BytesIO()
+        for chunk in r.iter_content(16_384):
+            buf.write(chunk)
+            if buf.tell() > CFG.pdf_size_cap:
+                raise RuntimeError("pdf exceeds cap")
+        return buf.getvalue()
+
+def _extract_pdf(buf: bytes) -> str:
+    try:
+        doc = fitz.open(stream=buf, filetype="pdf")
+    except Exception as e:
+        raise PDFExtractError(e)
+    parts, chars = [], 0
+    for page in doc:
+        if len(parts) >= CFG.pdf_pages_cap:
+            break
+        text = (
+            page.get_text("text")
+            .replace("\u00A0", " ")
+            .replace("-\n", "")
+        )
+        parts.append(text)
+        chars += len(text)
+        if chars > CFG.pdf_chars_cap:
+            break
+    out = " ".join(parts).strip()[:CFG.pdf_chars_cap]
+    if not out:
+        raise PDFExtractError("empty / scanned pdf")
+    return "[Retrieved from PyMUPDF]" + out
+
+def fetch_pdf(url: str) -> str:
+    try:
+        buf = _download_pdf(url)
+        return _extract_pdf(buf)
+    except Exception as e:
+        logging.error("PDF fetch failed for %s: %s", url, e)
+        return fetch_blocked_site(url)[:CFG.text_cap]

web_agents_5/fetchers/reddit_fetcher.py

@@ -0,0 +1,324 @@
+
+
+from __future__ import annotations
+from config import CFG, _SESS, _RND 
+import logging
+import re
+from bs4 import BeautifulSoup
+import functools
+import random
+import requests
+import time 
+import trafilatura
+from web_helpers import retry, fetch_blocked_site
+
+
+_REDDIT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
+
+
+@retry
+def _reddit_json_api(url: str) -> str | None:
+    api_url = re.sub(r"/comments/([a-z0-9]{6,8}).*", r"/comments/\1.json", url)
+    try:
+        headers = {"User-Agent": _REDDIT_UA, "Accept": "application/json"}
+        r = _SESS.get(
+            api_url,
+            params={"limit": 5, "depth": 2, "raw_json": 1},
+            headers=headers,
+            timeout=(CFG.connect_to, CFG.read_to),
+        )
+        if "blocked" in r.text.lower() or r.status_code != 200:
+            return None
+        
+        data = r.json()
+        post_data = data[0]["data"]["children"][0]["data"]
+        title = post_data.get("title", "")
+        selftext = post_data.get("selftext", "")
+        author = post_data.get("author", "")
+        
+        comments = []
+        if len(data) > 1:
+            for comment in data[1]["data"]["children"][:50]:
+                if comment["kind"] == "t1":
+                    c_author = comment["data"].get("author", "")
+                    c_body = comment["data"].get("body", "")
+                    if c_body:
+                        comments.append(f"u/{c_author}: {c_body}")
+        
+        result = f"Title: {title}\nPosted by: u/{author}\n\n"
+        if selftext:
+            result += f"{selftext}\n\n"
+        if comments:
+            result += "Top comments:\n\n" + "\n\n".join(comments)
+        
+        return result.strip()
+    except Exception:
+        return None
+import urllib.parse as _u
+
+_ID_RE = re.compile(r"([a-z0-9]{6,8})", re.I)
+
+def _extract_post_id(url: str) -> str | None:
+    """
+    Heuristics to find the 6–8‑char base‑36 Reddit ID in *any* post URL:
+      • short‑link  redd.it/<id>
+      • /r/sub/abc123/…               (old style)
+      • /comments/<id>/…              (API‑friendly)
+    """
+    # 1) short‑link host
+    u = _u.urlparse(url)
+    if u.netloc in {"redd.it", "www.redd.it"}:
+        return u.path.lstrip("/").split("/")[0] or None
+
+    # 2) /comments/<id>/ pattern (works already)
+    m = re.search(r"/comments/([a-z0-9]{6,8})", url, re.I)
+    if m:
+        return m.group(1)
+
+    # 3) generic “/r/<sub>/<id>/” or trailing “…/<id>”
+    parts = [p for p in u.path.split("/") if p]
+    for p in parts[::-1]:                       # search from right‑most
+        if _ID_RE.fullmatch(p):
+            return p
+    return None
+
+# ----------------------------------------------------------------------
+# Reddit OAuth helper – app‑only token (read‑only)
+# ----------------------------------------------------------------------
+import base64
+import threading
+
+_TOKEN_LOCK = threading.Lock()
+_REDDIT_TOKEN_CACHE: dict[str, float | str] = {"token": None, "expires": 0.0}
+
+def get_reddit_token(client_id: str, client_secret: str) -> str | None:
+    """
+    Return a cached bearer token obtained via Reddit's client‑credentials flow.
+    Returns None on error so callers can fall back to other scraping paths.
+    """
+    now = time.time()
+
+    # Fast path – cached and still valid
+    if (_tok := _REDDIT_TOKEN_CACHE["token"]) and now < _REDDIT_TOKEN_CACHE["expires"] - 30:
+        return _tok                       # 30‑sec grace
+
+    with _TOKEN_LOCK:                     # only one thread refreshes
+        # Re‑check after acquiring the lock
+        if (_tok := _REDDIT_TOKEN_CACHE["token"]) and now < _REDDIT_TOKEN_CACHE["expires"] - 30:
+            return _tok
+
+        try:
+            auth = requests.auth.HTTPBasicAuth(client_id, client_secret)
+            headers = {"User-Agent": _REDDIT_UA}
+            data = {"grant_type": "client_credentials"}  # app‑only, read‑only
+            r = requests.post(
+                "https://www.reddit.com/api/v1/access_token",
+                auth=auth,
+                data=data,
+                headers=headers,
+                timeout=10,
+            )
+            r.raise_for_status()
+            payload = r.json()
+            token  = payload["access_token"]
+            ttl    = int(payload.get("expires_in", 3600))
+            _REDDIT_TOKEN_CACHE.update({"token": token, "expires": now + ttl})
+            return token
+        except Exception as e:
+            logging.warning("Reddit token fetch failed: %s", e)
+            return None
+
+
+
+@retry
+def reddit_official_api(url: str, client_id: str, client_secret: str) -> str | None:
+    """
+    • Works for *any* Reddit permalink or short‑link.
+    • If the URL is a subreddit root (/r/<sub>) it still fetches 3 hot posts + top comment (unchanged).
+    """
+    token = get_reddit_token(client_id, client_secret)
+    if not token:
+        return None
+
+    headers = {
+        "Authorization": f"bearer {token}",
+        "User-Agent": _REDDIT_UA,
+    }
+
+    # ────────────────────────────────────────────────────────────────────
+    # 1.  Try to treat it as a *post* link by extracting an ID
+    # ────────────────────────────────────────────────────────────────────
+    post_id = _extract_post_id(url)
+    if post_id:
+        try:
+            r = requests.get(
+                f"https://oauth.reddit.com/comments/{post_id}",
+                headers=headers,
+                params={"limit": 5, "depth": 2, "raw_json": 1},
+                timeout=10,
+            )
+            r.raise_for_status()
+            data = r.json()
+
+            post   = data[0]["data"]["children"][0]["data"]
+            title  = post.get("title", "")
+            body   = post.get("selftext", "")
+            author = post.get("author", "")
+
+            comments = []
+            if len(data) > 1:
+                for c in data[1]["data"]["children"][:50]:
+                    if c["kind"] == "t1":
+                        c_auth = c["data"].get("author", "")
+                        c_body = c["data"].get("body", "")
+                        if c_body:
+                            comments.append(f"u/{c_auth}: {c_body}")
+
+            out = f"Title: {title}\nPosted by: u/{author}\n\n"
+            if body:
+                out += f"{body}\n\n"
+            if comments:
+                out += "Top comments:\n\n" + "\n\n".join(comments)
+            return out.strip()
+
+        except Exception as e:
+            logging.debug("Official API post fetch failed (%s); will try other strategies", e)
+
+    # ────────────────────────────────────────────────────────────────────
+    # 2.  If not a post (or the fetch above failed) treat as *subreddit*
+    #     root and list 3 hot posts, each with top comment (unchanged).
+    # ────────────────────────────────────────────────────────────────────
+    m_sub = re.search(r"reddit\.com/r/([^/?#]+)", url)
+    if not m_sub:
+        return None  # allow caller to fall back
+
+    subreddit = m_sub.group(1)
+    try:
+        r = requests.get(
+            f"https://oauth.reddit.com/r/{subreddit}/hot",
+            headers=headers,
+            params={"limit": 3, "raw_json": 1},
+            timeout=10,
+        )
+        r.raise_for_status()
+        posts = r.json()["data"]["children"]
+
+        out_blocks = []
+        for p in posts:
+            pd = p["data"]
+            pid   = pd["id"]
+            title = pd.get("title", "")
+            auth  = pd.get("author", "")
+            link  = pd.get("permalink", "")
+
+            # fetch one top comment
+            top_comment = ""
+            try:
+                c = requests.get(
+                    f"https://oauth.reddit.com/comments/{pid}",
+                    headers=headers,
+                    params={"limit": 1, "depth": 1, "raw_json": 1},
+                    timeout=10,
+                ).json()
+                if len(c) > 1:
+                    for cmt in c[1]["data"]["children"]:
+                        if cmt["kind"] == "t1":
+                            cauth = cmt["data"].get("author", "")
+                            cbody = cmt["data"].get("body", "")
+                            top_comment = f"u/{cauth}: {cbody}"
+                            break
+            except Exception:
+                pass
+
+            block = f"Title: {title}\nPosted by: u/{auth}\nLink: https://www.reddit.com{link}\n"
+            if top_comment:
+                block += f"Top comment:\n{top_comment}"
+            out_blocks.append(block)
+
+        return "\n\n---\n\n".join(out_blocks)
+
+    except Exception as e:
+        logging.debug("Official API subreddit fetch failed: %s", e)
+        return None
+
+
+@retry
+def _reddit_old_version(url: str) -> str | None:
+    old_url = url.replace("www.reddit.com", "old.reddit.com")
+    try:
+        r = _SESS.get(old_url, headers={"User-Agent": _REDDIT_UA}, timeout=(CFG.connect_to, CFG.read_to))
+        if r.status_code != 200:
+            return None
+        
+        soup = BeautifulSoup(r.text, "lxml")
+        title = soup.select_one(".title").text.strip() if soup.select_one(".title") else ""
+        author = soup.select_one(".author").text.strip() if soup.select_one(".author") else ""
+        post_body = soup.select_one(".usertext-body") 
+        post_text = post_body.get_text(strip=True) if post_body else ""
+        
+        comments = []
+        for comment in soup.select(".comment")[:50]:
+            c_author = comment.select_one(".author")
+            c_body = comment.select_one(".usertext-body")
+            if c_author and c_body:
+                comments.append(f"u/{c_author.text}: {c_body.get_text(strip=True)}")
+        
+        result = f"Title: {title}\nPosted by: u/{author}\n\n"
+        if post_text:
+            result += f"{post_text}\n\n"
+        if comments:
+            result += "Top comments:\n\n" + "\n\n".join(comments)
+        
+        return result.strip()
+    except Exception:
+        print("old reddit failed")
+        return None
+
+@retry
+def _pushshift_fallback(url: str) -> str | None:
+    m = re.search(r"/comments/([a-z0-9]{6,8})", url)
+    if not m:
+        return None
+    link_id = m.group(1)
+    try:
+        pst = _SESS.get(
+            "https://api.pushshift.io/reddit/submission/search/",
+            params={"ids": link_id, "size": 1},
+            timeout=10,
+        ).json()["data"]
+        post_txt = pst[0]["selftext"] if pst else ""
+        
+        com = _SESS.get(
+            "https://api.pushshift.io/reddit/comment/search/",
+            params={"link_id": link_id, "sort": "desc", "size": 3},
+            timeout=10,
+        ).json()["data"]
+        top_txt = "\n\n".join(c["body"] for c in com)
+        
+        txt = (post_txt + "\n\n" + top_txt).strip()
+        return txt or None
+    except Exception:
+        return None
+
+def fetch_reddit(url: str) -> str:
+    txt = _reddit_old_version(url)
+    if txt:
+        return "[Retrieved from Reddit]" + txt[:CFG.text_cap]
+
+    if CFG.reddit_client_id and CFG.reddit_client_secret:
+        # print("AAAA")
+        txt = reddit_official_api(url, CFG.reddit_client_id, CFG.reddit_client_secret)
+        if txt:
+            return "[Retrieved from Reddit]" +  txt[:CFG.text_cap]
+
+    txt = _reddit_json_api(url)
+    if txt:
+        return "[Retrieved from Reddit]" + txt[:CFG.text_cap]
+    
+    txt = _pushshift_fallback(url)
+    if txt:
+        return "[Retrieved from Reddit]" + txt[:CFG.text_cap]
+
+    
+    return fetch_blocked_site(url)[:CFG.text_cap]
+    

web_agents_5/fetchers/youtube_fetcher.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+import logging, re
+from config import CFG, _SESS
+from web_helpers import retry, fetch_blocked_site
+
+try:
+    import yt_dlp
+    _HAS = True
+except ImportError:
+    _HAS = False
+
+_LANGS = ["en", "en-US"]
+
+@retry
+def fetch_youtube(url: str) -> str:
+    if not _HAS:
+        return fetch_blocked_site(url)[:CFG.text_cap]
+
+    try:
+        ydl_opts = {"quiet": True, "no_warnings": True,
+                    "writesubtitles": True, "writeautomaticsub": True,
+                    "skip_download": True}
+        with yt_dlp.YoutubeDL(ydl_opts) as y:
+            info = y.extract_info(url, download=False)
+
+        subs = info.get("subtitles", {}) or {}
+        auto = info.get("automatic_captions", {}) or {}
+        tracks = next((subs.get(l) or auto.get(l) for l in _LANGS
+                       if subs.get(l) or auto.get(l)), None)
+        if not tracks:
+            tracks = next(iter(subs.values()), []) or next(iter(auto.values()), [])
+
+        if tracks:
+            cap_url = tracks[0]["url"]
+            if "fmt=" not in cap_url: cap_url += "&fmt=json3"
+            r = _SESS.get(cap_url, timeout=(CFG.connect_to, CFG.read_to))
+            r.raise_for_status()
+            if cap_url.endswith(".vtt"):
+                text = " ".join(l for l in r.text.splitlines()
+                                if l and "-->" not in l and not re.match(r"\d{2}:\d{2}", l))
+            else:
+                text = " ".join(seg["utf8"] for ev in r.json()["events"]
+                                for seg in ev.get("segs", []))
+            if text: return text[:CFG.text_cap]
+
+        meta = (info.get("title","") + "\n\n" + info.get("description","")).strip()
+        return "[Retrieved from yt-dlp] " + meta[:CFG.text_cap]
+    except Exception as e:
+        logging.error("YouTube fetch failed %s: %s", url, e)
+        return fetch_blocked_site(url)[:CFG.text_cap]

web_agents_5/fetchers_async.py

@@ -0,0 +1,155 @@
+# """
+# fetchers_async.py – Orchestrates multiple specialised fetchers **without changing
+# its public surface** (`async def fetch_url(url: str) -> str`).
+
+# Order of strategies (after specialised handlers):
+#     1. **Jina AI**         – fast & cheap full‑text extraction
+#     2. **Crawl4AI**        – browser‑based heavy‑weight fallback
+#     3. **Legacy HTML**     – trafilatura / readability last‑chance scrape
+
+# Specialised fetchers (PDF, YouTube, Reddit) remain unchanged.
+# """
+# from __future__ import annotations
+
+# import asyncio, logging
+# from typing import Callable
+
+# from web_helpers import retry
+# from fetchers.pdf_fetcher import fetch_pdf
+# from fetchers.youtube_fetcher import fetch_youtube
+# from fetchers.reddit_fetcher import fetch_reddit
+# from fetchers.github_fetcher import fetch_github
+
+# from fetchers.jina_fetcher import fetch_jina
+# from fetchers.crawl4ai_fetcher import fetch_crawl4ai
+# from fetchers.basic_fetcher import fetch_html
+
+
+# _ERR_PREFIXES = ("[error", "[failed", "[unable")
+
+
+# def _looks_error(txt: str | None) -> bool:
+#     return not txt or txt.strip().lower().startswith(_ERR_PREFIXES)
+
+
+# async def _thread_wrapper(fn: Callable[[str], str], url: str) -> str | None:
+#     try:
+#         return await asyncio.to_thread(fn, url)
+#     except Exception as exc:
+#         logging.debug("%s threw in thread: %s", fn.__name__, exc)
+
+# @retry
+# async def fetch_url(url: str) -> str:
+#     url_l = url.lower()
+    
+
+#     # 1 – Jina AI ------------------------------------------------------------
+#     if (out := await _thread_wrapper(fetch_jina, url)) and not _looks_error(out):
+#         return out
+    
+#     # if (out := await _thread_wrapper(fetch_html, url)) and not _looks_error(out):
+#     #     return out
+
+#     # 2 – Crawl4AI -----------------------------------------------------------
+#     try:
+#         md = await fetch_crawl4ai(url)
+#         if not _looks_error(md):
+#             return md
+#     except Exception as e:
+#         logging.debug("Crawl4AI failed: %s", e)
+        
+#     if "pdf" in url_l:
+#         if (out := await _thread_wrapper(fetch_pdf, url)) and not _looks_error(out):
+#             return out
+        
+#     if "reddit" in url_l:
+#         if (out := await _thread_wrapper(fetch_reddit, url)) and not _looks_error(out):
+#             return out
+#     if "youtube" in url_l:
+#         if (out := await _thread_wrapper(fetch_youtube, url)) and not _looks_error(out):
+#             return out
+#     if "github" in url_l:
+#         if (out := await _thread_wrapper(fetch_github, url)) and not _looks_error(out):
+#             return out
+
+#     # 3 – Basic HTML --------------------------------------------------------
+#     if (out := await _thread_wrapper(fetch_html, url)) and not _looks_error(out):
+#         return out
+
+#     return "[error fetch_url exhausted all methods]"
+
+
+
+import asyncio, logging, time
+
+from fetchers.pdf_fetcher     import fetch_pdf
+from fetchers.youtube_fetcher import fetch_youtube
+from fetchers.reddit_fetcher  import fetch_reddit
+from fetchers.github_fetcher  import fetch_github
+from fetchers.jina_fetcher    import fetch_jina
+from fetchers.crawl4ai_fetcher import fetch_crawl4ai
+from fetchers.basic_fetcher   import fetch_html
+
+_ERR_PREFIXES = ("[error", "[failed", "[unable]")
+
+def _looks_error(txt: str | None) -> bool:
+    return not txt or txt.strip().lower().startswith(_ERR_PREFIXES)
+
+# per-fetcher hard caps (seconds)
+_FETCHER_TIMEOUTS = {
+    "fetch_jina":      20.0,
+    "fetch_github":    10.0,
+    "fetch_crawl4ai":  40.0,
+    "fetch_html":      20.0,
+    "fetch_pdf":       30.0,
+    "fetch_youtube":   30.0,
+    "fetch_reddit":    10.0,
+}
+
+
+async def fetch_url(url: str) -> str:
+    url_l = url.lower()
+
+    async def timed_fetch(fn) -> str | None:
+        name     = fn.__name__
+        timeout  = _FETCHER_TIMEOUTS.get(name, 60.0)
+        start_ts = time.perf_counter()
+        try:
+            # choose sync or async execution path
+            coro = fn(url) if asyncio.iscoroutinefunction(fn) else asyncio.to_thread(fn, url)
+            result = await asyncio.wait_for(coro, timeout=timeout)
+            elapsed = (time.perf_counter() - start_ts) * 1000
+            if result and not _looks_error(result):
+                logging.info(f"[{name}] ✅ success in {elapsed:.1f} ms")
+                return result
+            logging.warning(f"[{name}] ❌ error response in {elapsed:.1f} ms")
+        except asyncio.TimeoutError:
+            logging.warning(f"[{name}] ⏱️ timed-out after {timeout}s")
+        except Exception as e:
+            elapsed = (time.perf_counter() - start_ts) * 1000
+            logging.warning(f"[{name}] 💥 exception in {elapsed:.1f} ms → {e}")
+        return None
+
+    async def try_chain(*fetchers) -> str | None:
+        for fn in fetchers:
+            if result := await timed_fetch(fn):
+                return result
+        return None
+
+    # -------------- domain-specific chains ---------------
+    if "github.com"   in url_l:
+        return await try_chain(fetch_jina, fetch_github,  fetch_crawl4ai)
+    if "wikipedia.org" in url_l:
+        return await try_chain(fetch_html, fetch_jina,     fetch_crawl4ai)
+    if "reddit.com"   in url_l:
+        return await try_chain(fetch_jina, fetch_reddit,   fetch_html)
+    if "quora.com"    in url_l:
+        return await try_chain(fetch_crawl4ai, fetch_jina, fetch_html)
+    if "youtube.com"  in url_l or "youtu.be" in url_l:
+        return await try_chain(fetch_jina, fetch_youtube)
+    if url_l.endswith(".pdf") or "pdf" in url_l:
+        return await try_chain(fetch_jina, fetch_pdf, fetch_html, fetch_crawl4ai)
+
+    # -------------- generic fallback ---------------------
+    return (await try_chain(fetch_jina, fetch_crawl4ai, fetch_html)
+            or "[error fetch_url exhausted all methods]")

web_agents_5/host_serper2.sh

@@ -0,0 +1,6 @@
+export PYTHONPATH=./
+export MAX_OUTBOUND=256
+export JINA_CACHE_DIR=./../.cache/jina_cache
+export SERPER_CACHE_DIR=./../.cache/serper_cache
+
+python ./web_agents_5/sandbox_serper.py --port $PORT_SERPER_HOST --workers 256

web_agents_5/sandbox_serper.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""sandbox_serper.py – resilient Serper sandbox v2.1
+
+Fixes
+-----
+* Moved `global _MAX_OUTBOUND, _SEM` declaration to the **top of `main()`**
+  before any reference, eliminating the `SyntaxError: name used prior to
+  global declaration`.
+* No functional changes otherwise.
+"""
+
+from __future__ import annotations
+import argparse, asyncio, logging, os, time, traceback
+from fastapi import FastAPI
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel
+import uvicorn
+import time
+# ───────────────────────── logging setup ──────────────────────────
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger("sandbox_serper")
+
+app = FastAPI()
+
+class Req(BaseModel):
+    env: str
+    call: str
+    timeout: int = 60
+
+# ───────────────────── global throughput gate ─────────────────────
+_MAX_OUTBOUND = int(os.getenv("MAX_OUTBOUND", "10"))
+_SEM = asyncio.Semaphore(_MAX_OUTBOUND)
+
+# ───────────────────────── endpoint ───────────────────────────────
+@app.post("/execute")
+async def execute(req: Req):
+    # async with _SEM:
+    async with _SEM:          #  ❰❰  throttle
+        result = await run_in_threadpool(_safe_eval, req.env,
+                                          req.call, req.timeout)
+
+    return {
+        "output": "",
+        "result": result,
+        "error": None if not str(result).startswith("[tool-error]") else result,
+    }
+
+# ───────────────────── sandbox evaluator ──────────────────────────
+
+def _safe_eval(env: str, call: str, timeout: int):
+    start = time.time(); loc: dict = {}
+    try:
+        exec(env, {}, loc)
+        exec(f"response = {call}", {}, loc)
+        if time.time() - start > timeout:
+            raise TimeoutError(f"wall-clock timeout for call {call}")
+        return loc.get("response", "[tool-error] no response var")
+    except Exception as e:
+        log.error("tool error: %s\n%s", e, traceback.format_exc())
+        return f"[tool-error] {e}"
+
+# ─────────────────────────── main ────────────────────────────────
+
+def main():
+    global _MAX_OUTBOUND, _SEM  # ← moved to top
+
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--port", type=int, default=1211)
+    ap.add_argument("--workers", type=int, default=1)
+    ap.add_argument("--reload", action="store_true")
+    # ap.add_argument("--max_outbound", type=int, default=_MAX_OUTBOUND,
+                    # help="simultaneous outbound calls across all workers")
+    args = ap.parse_args()
+
+    _SEM = asyncio.Semaphore(_MAX_OUTBOUND)
+
+    if args.reload and args.workers > 1:
+        raise SystemExit("--reload and --workers>1 are mutually exclusive")
+
+    # log.info("Starting sandbox :%d | workers=%d | max_outbound=%d",
+            #  args.port, args.workers, _MAX_OUTBOUND)
+
+    if args.workers > 1:
+        uvicorn.run("sandbox_serper:app", host="0.0.0.0", port=args.port, workers=args.workers)
+    else:
+        uvicorn.run(app, host="0.0.0.0", port=args.port, reload=args.reload)
+
+
+if __name__ == "__main__":
+    main()

web_agents_5/search_api.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+import asyncio
+from typing import Dict, List
+from utils import (google_search, url_hits_to_markdown,
+                   search_result_to_markdown,
+                   async_search_and_extract, _bad)
+from fetchers_async import fetch_url
+from compressor import compress_text, query_text
+from config import CFG
+import logging
+
+# print("PYTHONPATH:", sys.path)
+
+def web_search(query):
+    return search_urls(query = query, top_k = 10)
+
+def web_visit(url):
+    return open_url(url = url, compress = False)
+
+# ── 1. search_urls ──────────────────────────────────────────────────────
+def search_urls(query: str, top_k: int = 10) -> str:
+    return url_hits_to_markdown(google_search(query, top_k))
+
+# ── 2. open_url ─────────────────────────────────────────────────────────
+def open_url(url: str, *, compress: bool = True, pct: float = CFG.pct,
+             model: str = "gpt-4o-mini") -> str:
+    if _bad(url): return _bad(url)
+    try:
+        body = asyncio.run(fetch_url(url))
+        body = str(body)
+    except Exception as e:
+        return f"[error fetching URL: {e}]"
+    if compress:
+        try:
+            body = compress_text(body, pct=pct, model=model)
+        except Exception as e:
+            body = f"[compression failed: {e}]\n\n{body[:2000]}"
+    return body
+
+# ── 3. search_and_parse_query ───────────────────────────────────────────
+def search_and_parse_query(query: str, top_k: int = 3, *,
+                           compress: bool = True, pct: float = CFG.pct) -> str:
+    blocks = asyncio.run(async_search_and_extract(query, top_k))
+    if compress:
+        for b in blocks:
+            try:
+                cmp = compress_text(b["body"], pct=pct)
+                b["body"] = (f"**Summary:**\n{cmp['narrative']}\n\n"
+                             f"**Facts:**\n{cmp['facts']}\n\n"
+                             f"**Tables:**\n{cmp['tables']}")
+            except Exception as e:
+                b["body"] = f"[compression failed: {e}]\n\n{b['body']}"
+    return search_result_to_markdown(blocks)
+
+# ── 4. query_url ────────────────────────────────────────────────────────
+def query_url(url: str, goal: str, *, model: str = "gpt-4.1-mini") -> str:
+    if _bad(url): return _bad(url)
+    body = asyncio.run(fetch_url(url))
+    if not body or body.startswith("[error"):
+        return f"[failed to retrieve content from {url}]\n\n{body}"
+    return query_text(url, body, goal, model=model)['extracted_info']
+    
+

web_agents_5/utils.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+import asyncio, logging, re, tiktoken
+from typing import Dict, List
+from config import CFG, _SESS
+from fetchers_async import fetch_url
+from web_helpers import retry
+from urllib.parse import urlparse
+
+enc = tiktoken.get_encoding("cl100k_base")
+
+# ── Google / Serper search ──────────────────────────────────────────────
+
+# def google_search(query: str, top_k: int = 10) -> List[Dict[str,str]]:
+#     if not CFG.serper_key:
+#         raise EnvironmentError("SERPER_API_KEY not set")
+#     r = _SESS.post(
+#         CFG.serper_ep,
+#         headers={"X-API-KEY": CFG.serper_key, "Content-Type": "application/json"},
+#         json={"q": query}, timeout=20)
+#     r.raise_for_status()
+#     hits = []
+#     for it in r.json().get("organic", []):
+#         hits.append({"title": it.get("title",""),
+#                      "link":  it.get("link",""),
+#                      "snippet": it.get("snippet","")})
+#         if len(hits) == top_k: break
+#     return hits
+import hashlib, json, logging, os, time
+from typing import List, Dict
+
+def _canon_query(q: str) -> str:
+    # Normalize whitespace to avoid duplicate keys for e.g. "foo  bar"
+    return " ".join((q or "").strip().split())
+
+
+def _search_cache_key(query: str, top_k: int) -> str:
+    cq = _canon_query(query)
+    raw = f"{top_k}|{cq}"
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest() + ".json"
+
+def _search_cache_paths(query: str, top_k: int) -> str:
+    root = CFG.serper_cache_dir
+    os.makedirs(root, exist_ok=True)
+    return os.path.join(root, _search_cache_key(query, top_k))
+
+def _ttl_seconds() -> int:
+    # 0 or missing → no expiry
+    try:
+        return int(getattr(CFG, "search_cache_ttl", 0) or int(os.environ.get("SEARCH_CACHE_TTL", "0")))
+    except Exception:
+        return 0
+
+def _load_search_cache(path: str) -> List[Dict[str, str]] | None:
+    try:
+        if not os.path.exists(path) or os.path.getsize(path) <= 2:
+            return None
+        ttl = _ttl_seconds()
+        if ttl > 0:
+            age = time.time() - os.path.getmtime(path)
+            if age > ttl:
+                return None
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        # Basic shape check: list of dicts with expected keys
+        if isinstance(data, list):
+            return data
+    except Exception as e:
+        logging.debug("Serper cache read failed (%s): %s", path, e)
+    return None
+
+def _save_search_cache(path: str, hits: List[Dict[str, str]]) -> None:
+    try:
+        tmp = f"{path}.tmp.{os.getpid()}"
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(hits, f, ensure_ascii=False)
+        os.replace(tmp, path)  # atomic on same FS
+    except Exception as e:
+        logging.debug("Serper cache write failed (%s): %s", path, e)
+        
+    
+@retry
+def google_search(query: str, top_k: int = 10) -> List[Dict[str,str]]:
+    if not CFG.serper_key:
+        raise EnvironmentError("SERPER_API_KEY not set")
+
+    cpath = _search_cache_paths(query, top_k)
+    cached = _load_search_cache(cpath)
+    if cached is not None:
+        logging.info("Serper search (cache hit) ← %r (top_k=%d)", _canon_query(query), top_k)
+        return cached
+
+    r = _SESS.post(
+        CFG.serper_ep,
+        headers={"X-API-KEY": CFG.serper_key, "Content-Type": "application/json"},
+        json={"q": query},
+        timeout=20
+    )
+    r.raise_for_status()
+    hits: List[Dict[str, str]] = []
+    for it in r.json().get("organic", []):
+        hits.append({
+            "title": it.get("title", ""),
+            "link": it.get("link", ""),
+            "snippet": it.get("snippet", ""),
+        })
+        if len(hits) == top_k:
+            break
+
+    _save_search_cache(cpath, hits)
+    return hits
+
+
+# ── async extract per hit ───────────────────────────────────────────────
+async def async_search_and_extract(query: str, top_k: int = 5) -> List[Dict]:
+    hits = google_search(query, top_k)
+    async def enrich(h):
+        return {**h, "body": await fetch_url(h["link"])}
+    return await asyncio.gather(*(enrich(h) for h in hits))
+
+# ── markdown helpers ────────────────────────────────────────────────────
+def url_hits_to_markdown(hits: List[Dict[str,str]]) -> str:
+    out = []
+    for i, h in enumerate(hits, 1):
+        out.append(f"### {i}. {h['title']}\n**URL**: {h['link']}\n\n**Snippet**: {h['snippet']}\n")
+    return "\n---\n\n".join(out)
+
+def search_result_to_markdown(blocks: List[Dict]) -> str:
+    out = []
+    for i, b in enumerate(blocks, 1):
+        out.append(f"### {i}. **Title**: {b['title']}\n**URL**: {b['link']}\n\n"
+                   f"**Snippet**: {b['snippet']}\n\n**Content**:\n{b['body']}\n")
+    return "\n---\n\n".join(out)
+
+def trim_to_tokens(text: str, limit: int, model: str = "gpt-3.5-turbo") -> str:
+    ids = enc.encode(text)
+    if len(ids) <= limit: return text
+    keep = limit // 2
+    return enc.decode(ids[:keep] + ids[-keep:])
+
+def _bad(url: str) -> str|None:
+    p = urlparse(url)
+    if p.scheme not in ("http","https") or not p.netloc:
+        return "[error: invalid URL – must start with http:// or https://]"
+    return None
+

web_agents_5/web_helpers.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+import functools, logging, random, re, time, requests, trafilatura
+from typing import Callable
+from bs4 import BeautifulSoup
+from config import CFG, _RND
+
+# ── retry ────────────────────────────────────────────────────────────────
+def retry(fn: Callable) -> Callable:
+    @functools.wraps(fn)
+    def _wrap(*a, **kw):
+        for i in range(CFG.retries):
+            try:
+                return fn(*a, **kw)
+            except Exception as e:
+                if i == CFG.retries - 1:
+                    raise
+                delay = CFG.backoff * (2 ** i) * (1 + 0.3 * _RND.random())
+                logging.warning("Retry %s/%s %s: %s (%.2fs)",
+                                i+1, CFG.retries, fn.__name__, e, delay)
+                time.sleep(delay)
+    return _wrap
+
+# ── text extraction ──────────────────────────────────────────────────────
+def extract_main_text(html: str) -> str:
+    txt = trafilatura.extract(html, output_format="txt") or ""
+    if len(txt) >= 500:
+        return txt
+    from readability import Document
+    soup = BeautifulSoup(Document(html).summary(), "lxml")
+    txt  = soup.get_text(" ", strip=True)
+    if len(txt) >= 400:
+        return txt
+    for tag in soup(["script", "style", "noscript"]):
+        tag.decompose()
+    return re.sub(r"\s+", " ", soup.get_text(" ").strip())
+
+# ── last‑chance fetch when everything fails ──────────────────────────────
+@retry
+def fetch_blocked_site(url: str) -> str:
+    hdrs = {"User-Agent": CFG.ua, "Referer": "https://www.google.com/"}
+    sess = requests.Session(); sess.headers.update(hdrs)
+
+    # 1) direct
+    try:
+        r = sess.get(url, timeout=(CFG.connect_to, CFG.read_to))
+        r.raise_for_status()
+        txt = extract_main_text(r.text)
+        if len(txt) > 500:
+            return "[Retrieved from redirected attempt]\n\n" + txt[:CFG.text_cap]
+    except Exception as e:
+        logging.debug("Direct scrape failed %s: %s", url, e)
+
+    # 2) wayback
+    try:
+        wb = f"https://web.archive.org/web/2023/{url}"
+        r  = sess.get(wb, timeout=(CFG.connect_to, CFG.read_to))
+        r.raise_for_status()
+        txt = extract_main_text(r.text)
+        if len(txt) > 500:
+            return "[Retrieved from archive.org]\n\n" + txt[:CFG.text_cap]
+    except Exception as e:
+        logging.debug("Wayback scrape failed %s: %s", url, e)
+
+    return f"[Error accessing {url}. Try VPN or manual archive.org check.]"