Update main.py

main.py

@@ -1,7 +1,7 @@
 import os
 import asyncio
 from fastapi import FastAPI, HTTPException
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, create_model
 from typing import List, Optional
 from crawl4ai import AsyncWebCrawler
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy, LLMExtractionStrategy
@@ -31,15 +31,22 @@ async def crawl(input: CrawlerInput):
     # Create a dictionary with columns as keys and descriptions as values
     extraction_info = {col: desc for col, desc in zip(input.columns, input.descriptions)}
     
-    # Convert the dictionary to a JSON string
+    # Create a dynamic Pydantic model based on the input columns and descriptions
+    dynamic_model = create_model(
+        'DynamicModel',
+        **{col: (str, Field(..., description=desc)) for col, desc in extraction_info.items()}
+    )
+    
+    # Convert the dictionary to a JSON string for the instruction
     instruction = f"Extract the following information: {json.dumps(extraction_info)}"
     
     async with AsyncWebCrawler(verbose=True) as crawler:
         result = await crawler.arun(
             url=input.url,
             extraction_strategy=LLMExtractionStrategy(
-                provider="openai/gpt-4o-mini",
+                provider="openai/gpt-3.5-turbo",
                 api_token=os.getenv('OPENAI_API_KEY'),
+                schema=dynamic_model.schema(),
                 extraction_type="schema",
                 verbose=True,
                 instruction=instruction
@@ -56,4 +63,4 @@ async def test():
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=7860)
+    uvicorn.run(app, host="0.0.0.0", port=8000)