include markitdown

app.py

@@ -21,11 +21,12 @@ LANGFUSE_PUBLIC_KEY = os.environ.get("LANGFUSE_PUBLIC_KEY")
 LANGFUSE_SECRET_KEY = os.environ.get("LANGFUSE_SECRET_KEY")
 
 LANGFUSE_AUTH = base64.b64encode(
-    f"{LANGFUSE_PUBLIC_KEY}:{LANGFUSE_SECRET_KEY}".encode()
-).decode()
+    f"{LANGFUSE_PUBLIC_KEY}:{LANGFUSE_SECRET_KEY}".encode()).decode()
 
-os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get("LANGFUSE_HOST") + "/api/public/otel"
-os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
+os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get(
+    "LANGFUSE_HOST") + "/api/public/otel"
+os.environ[
+    "OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
 
 # Create a TracerProvider for OpenTelemetry
 trace_provider = TracerProvider()
@@ -35,6 +36,7 @@ trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter()))
 
 # Set the global default tracer provider
 from opentelemetry import trace
+
 trace.set_tracer_provider(trace_provider)
 tracer = trace.get_tracer(__name__)
 
@@ -64,9 +66,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
 
   # 1. Instantiate Agent ( modify this part to create your agent)
   try:
-    print("This is new code creating the new basic_agent")
     agent = basic_agent.BasicAgent()
-    print("This is new code creating the new basic_agent is now done")
   except Exception as e:
     print(f"Error instantiating agent: {e}")
     return f"Error initializing agent: {e}", None
@@ -102,6 +102,14 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
   for item in questions_data:
     task_id = item.get("task_id")
     question_text = item.get("question")
+    filename = item.get("file_name")
+    if filename:
+      file_url = f"{api_url}/files/{task_id}"
+      print("Question {task_id=} has attachment: {file_url=}")
+      quest_with_url = f"{question_text}\nFile url: {file_url}"
+    else:
+      quest_with_url = question_text
+
     if not task_id or question_text is None:
       print(f"Skipping item with missing task_id or question: {item}")
       continue

basic_agent.py

@@ -3,6 +3,7 @@ import logging
 import sys
 import time
 
+import markitdown
 import smolagents
 
 # Logs appear to be swallowed.
@@ -23,9 +24,34 @@ Remember - The FINAL ANSWER must be a string, a number or a comma separated list
 """
 
 
+@smolagents.tool
+def convert_from_url(url: str) -> str:
+  """Convert a file at a url into a markdown string.
+
+   This method takes a path to a file and converts it into
+   a markdown string. It can handle a variety of file
+   types as input including pdf, excel, xlsx, mp3, csv,
+   powerpoint, etc.
+
+   Args:
+     url: the url to access the file. If the file is on the
+          local machine, the url should start with "file://"
+          instead of the traditional "http://" or "https://".
+
+   Returns:
+     string that is in the markdown format.
+   """
+  # Don't need third-party plugins.
+  md = markitdown.MarkItDown(enable_plugins=Disable)
+  result = md.convert(url)
+  return result.text_content
+
+
 class BasicAgent:
+
   def check_results(self, final_answer, agent_memory):
-    if isinstance(final_answer, str) or isinstance(final_answer, int) or isinstance(final_answer, float):
+    if isinstance(final_answer, str) or isinstance(
+        final_answer, int) or isinstance(final_answer, float):
       return True
     failure = "FAILED! The final answer must be a string, a number or a comma separated list of strings and numbers."
     print(f"Feedback: {failure}")
@@ -34,10 +60,10 @@ class BasicAgent:
   def __init__(self):
     print("BasicAgent initializing.")
 
-# too big
-#    self.manager_model_id = "google/gemma-3-27B-it"
-#    self.manager_model_id = "OpenGVLab/InternVL3-14B"
-#    self.manager_model_id = "AIDC-AI/Ovis2-34B"
+    # too big
+    #    self.manager_model_id = "google/gemma-3-27B-it"
+    #    self.manager_model_id = "OpenGVLab/InternVL3-14B"
+    #    self.manager_model_id = "AIDC-AI/Ovis2-34B"
 
     self.manager_model_id = "meta-llama/Llama-3.3-70B-Instruct"
     self.manager_model = None
@@ -61,13 +87,32 @@ class BasicAgent:
           custom_role_conversions=None,
       )
 
-    print(f"NEW4: BasicAgent {self.manager_model_id=} {self.code_model_id=}")
+    print(f"NEW5: BasicAgent {self.manager_model_id=} {self.code_model_id=}")
 
+    file_convert_tool = [
+        convert_from_url,
+        smolagents.FinalAnswerTool(),
+    ]
+    self.convert_files_to_markdown = smolagents.CodeAgent(
+        name="convert_files_to_markdown",
+        description=
+        "Given a url, it converts the file from various formats (such as pdf, excel, powerpoint, audio, csv, etc) into the markdown format",
+        model=self.code_model,
+        tools=file_convert_tool,
+        max_steps=6,
+        verbosity_level=2,
+        planning_interval=None,
+        additional_authorized_imports=["markitdown"],
+    )
+
+    math_tools = [
+        smolagents.FinalAnswerTool(),
+    ]
     self.math_agent = smolagents.CodeAgent(
         name="date_time_math_agent",
         description="Handle date, time and math caclulations",
         model=self.code_model,
-        tools=[],
+        tools=math_tools,
         max_steps=6,
         verbosity_level=2,
         planning_interval=None,
@@ -117,10 +162,17 @@ class BasicAgent:
         max_steps=10,
         verbosity_level=2,
         planning_interval=None,
-        additional_authorized_imports=["duckduckgo_search", "wikipedia-api"],
-        managed_agents=[self.web_search_agent, self.wikipedia_search_agent, self.math_agent],
+        additional_authorized_imports=[
+            "duckduckgo_search", "wikipedia-api", "markitdown"
+        ],
+        managed_agents=[
+            self.web_search_agent,
+            self.wikipedia_search_agent,
+            self.math_agent,
+            self.convert_files_to_markdown,
+        ],
         final_answer_checks=[self.check_results],
-        )
+    )
 
   def __call__(self, question: str) -> str:
     print(f"Agent received question (first 50 chars): {question[:50]}...")

basic_agent_test.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
 import logging
+import json
 
 import basic_agent
 
 LOG = logging.getLogger(__name__)
 
+QUESTIONS_PATH = "../questions.json"
 
 # Local:
 #  https://huggingface.co/docs/smolagents/tutorials/inspect_runs
@@ -27,18 +29,44 @@ from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProces
 
 endpoint = "http://0.0.0.0:6006/v1/traces"
 trace_provider = TracerProvider()
-trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint)))
+trace_provider.add_span_processor(
+    SimpleSpanProcessor(OTLPSpanExporter(endpoint)))
 
 SmolagentsInstrumentor().instrument(tracer_provider=trace_provider)
 
 ba = basic_agent.BasicAgent()
 
-"""
-answer = ba(
-    "Who is the 47th president of the united states? If necessary, use a web search to get the most up to date information."
-)
-"""
-answer = ba("What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?"
-)
+with open(QUESTIONS_PATH, "r") as fp:
+  questions = json.load(fp)
+
+# Should be 20
+print(f"{len(questions)=}")
+
+# One question entry:
+#
+# {'Level': '1',
+#  'file_name': '7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx',
+#  'question': 'The attached Excel file contains the sales of menu items for a '
+#              'local fast-food chain. What were the total sales that the chain '
+#              'made from food (not including drinks)? Express your answer in '
+#              'USD with two decimal places.',
+#  'task_id': '7bd855d8-463d-4ed5-93ca-5fe35145f733'},
+#
+
+want = 7
+question = questions[want]
+
+q_task_id = question["task_id"]
+q_text = question["question"]
+q_filename = question.get("file_name", "")
+
+if q_filename:
+  file_url = f"{api_url}/files/{task_id}"
+  print("Question {task_id=} has attachment: {file_url=}")
+  quest_with_url = f"{q_text}\nFile url: {file_url}"
+else:
+  quest_with_url = q_text
+
+answer = ba(question_text)
 
 LOG.warning(f"{answer=}")

requirements.txt

@@ -1,6 +1,7 @@
 gradio
 requests
 huggingface_hub[hf_xet]
+markitdown[all]
 smolagents
 smolagents[openai]
 wikipedia-api