chore: initialize backend project structure with FastAPI

backend/app/__init__.py

@@ -0,0 +1,3 @@
+"""ScrapeRL Backend - FastAPI-based RL environment for web scraping."""
+
+__version__ = "0.1.0"

backend/app/config.py

@@ -0,0 +1,98 @@
+"""Configuration management using Pydantic Settings."""
+
+from functools import lru_cache
+from typing import Literal
+
+from pydantic import Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Application settings loaded from environment variables."""
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    # Application
+    app_name: str = "ScrapeRL"
+    app_version: str = "0.1.0"
+    debug: bool = Field(default=False, description="Enable debug mode")
+    log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
+
+    # Server
+    host: str = "0.0.0.0"
+    port: int = 8000
+    reload: bool = False
+    workers: int = 1
+
+    # CORS
+    cors_origins: list[str] = Field(
+        default=["http://localhost:3000", "http://localhost:5173"],
+        description="Allowed CORS origins",
+    )
+    cors_allow_credentials: bool = True
+    cors_allow_methods: list[str] = ["*"]
+    cors_allow_headers: list[str] = ["*"]
+
+    # LLM Providers
+    openai_api_key: SecretStr | None = Field(default=None, description="OpenAI API key")
+    anthropic_api_key: SecretStr | None = Field(default=None, description="Anthropic API key")
+    google_api_key: SecretStr | None = Field(default=None, description="Google AI API key")
+    groq_api_key: SecretStr | None = Field(default=None, description="Groq API key")
+
+    # Model Defaults
+    default_model: str = "gpt-4o-mini"
+    default_temperature: float = 0.7
+    max_tokens: int = 4096
+
+    # Search Providers
+    google_search_api_key: SecretStr | None = None
+    google_search_engine_id: str | None = None
+    bing_search_api_key: SecretStr | None = None
+
+    # ChromaDB
+    chroma_persist_directory: str = "./data/chroma"
+    chroma_collection_name: str = "scraperl_memory"
+
+    # Episode Settings
+    max_steps_per_episode: int = 50
+    default_timeout_seconds: float = 30.0
+
+    # Browser Settings
+    headless_browser: bool = True
+    browser_timeout_ms: int = 30000
+
+    # Memory Settings
+    short_term_memory_size: int = 100
+    working_memory_size: int = 20
+    long_term_memory_top_k: int = 10
+
+    # Reward Weights
+    reward_accuracy_weight: float = 0.4
+    reward_efficiency_weight: float = 0.2
+    reward_cost_weight: float = 0.2
+    reward_completeness_weight: float = 0.2
+
+    @property
+    def available_providers(self) -> list[str]:
+        """Return list of configured LLM providers."""
+        providers = []
+        if self.openai_api_key:
+            providers.append("openai")
+        if self.anthropic_api_key:
+            providers.append("anthropic")
+        if self.google_api_key:
+            providers.append("google")
+        if self.groq_api_key:
+            providers.append("groq")
+        return providers
+
+
+@lru_cache
+def get_settings() -> Settings:
+    """Get cached settings instance."""
+    return Settings()

backend/app/main.py

@@ -0,0 +1,140 @@
+"""FastAPI application entry point with CORS and lifespan management."""
+
+import logging
+from contextlib import asynccontextmanager
+from typing import AsyncGenerator
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.api.routes import agents, episode, health, memory, tasks, tools
+from app.config import get_settings
+from app.memory.manager import MemoryManager
+from app.models.router import SmartModelRouter
+from app.tools.registry import MCPToolRegistry
+from app.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+# Global instances for dependency injection
+_memory_manager: MemoryManager | None = None
+_model_router: SmartModelRouter | None = None
+_tool_registry: MCPToolRegistry | None = None
+
+
+def get_memory_manager() -> MemoryManager:
+    """Get the global memory manager instance."""
+    if _memory_manager is None:
+        raise RuntimeError("Memory manager not initialized")
+    return _memory_manager
+
+
+def get_model_router() -> SmartModelRouter:
+    """Get the global model router instance."""
+    if _model_router is None:
+        raise RuntimeError("Model router not initialized")
+    return _model_router
+
+
+def get_tool_registry() -> MCPToolRegistry:
+    """Get the global tool registry instance."""
+    if _tool_registry is None:
+        raise RuntimeError("Tool registry not initialized")
+    return _tool_registry
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
+    """Manage application lifespan - startup and shutdown events."""
+    global _memory_manager, _model_router, _tool_registry
+
+    settings = get_settings()
+    logger.info(f"Starting {settings.app_name} v{settings.app_version}")
+
+    # Initialize components
+    logger.info("Initializing memory manager...")
+    _memory_manager = MemoryManager(settings)
+    await _memory_manager.initialize()
+
+    logger.info("Initializing model router...")
+    _model_router = SmartModelRouter(settings)
+    await _model_router.initialize()
+
+    logger.info("Initializing tool registry...")
+    _tool_registry = MCPToolRegistry()
+    await _tool_registry.initialize()
+
+    logger.info("Application startup complete")
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down application...")
+
+    if _memory_manager:
+        await _memory_manager.shutdown()
+    if _model_router:
+        await _model_router.shutdown()
+    if _tool_registry:
+        await _tool_registry.shutdown()
+
+    logger.info("Application shutdown complete")
+
+
+def create_app() -> FastAPI:
+    """Create and configure the FastAPI application."""
+    settings = get_settings()
+    setup_logging(settings.log_level)
+
+    app = FastAPI(
+        title=settings.app_name,
+        description="FastAPI-based RL environment for intelligent web scraping",
+        version=settings.app_version,
+        debug=settings.debug,
+        lifespan=lifespan,
+        docs_url="/docs",
+        redoc_url="/redoc",
+        openapi_url="/openapi.json",
+    )
+
+    # Configure CORS
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.cors_origins,
+        allow_credentials=settings.cors_allow_credentials,
+        allow_methods=settings.cors_allow_methods,
+        allow_headers=settings.cors_allow_headers,
+    )
+
+    # Include routers
+    api_prefix = "/api"
+    app.include_router(health.router, prefix=api_prefix, tags=["Health"])
+    app.include_router(episode.router, prefix=api_prefix, tags=["Episode"])
+    app.include_router(tasks.router, prefix=api_prefix, tags=["Tasks"])
+    app.include_router(agents.router, prefix=api_prefix, tags=["Agents"])
+    app.include_router(tools.router, prefix=api_prefix, tags=["Tools"])
+    app.include_router(memory.router, prefix=api_prefix, tags=["Memory"])
+
+    return app
+
+
+# Create the application instance
+app = create_app()
+
+
+def run() -> None:
+    """Run the application using uvicorn."""
+    settings = get_settings()
+    uvicorn.run(
+        "app.main:app",
+        host=settings.host,
+        port=settings.port,
+        reload=settings.reload,
+        workers=settings.workers,
+        log_level=settings.log_level.lower(),
+    )
+
+
+if __name__ == "__main__":
+    run()

backend/pyproject.toml

@@ -0,0 +1,81 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scraperl-backend"
+version = "0.1.0"
+description = "FastAPI-based RL environment for intelligent web scraping"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.11"
+authors = [
+    { name = "ScrapeRL Team" }
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Framework :: FastAPI",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "fastapi>=0.109.0",
+    "uvicorn[standard]>=0.27.0",
+    "pydantic>=2.5.0",
+    "pydantic-settings>=2.1.0",
+    "httpx>=0.26.0",
+    "chromadb>=0.4.22",
+    "beautifulsoup4>=4.12.0",
+    "lxml>=5.1.0",
+    "openai>=1.10.0",
+    "anthropic>=0.18.0",
+    "google-generativeai>=0.4.0",
+    "groq>=0.4.0",
+    "playwright>=1.41.0",
+    "tiktoken>=0.5.0",
+    "numpy>=1.26.0",
+    "tenacity>=8.2.0",
+    "structlog>=24.1.0",
+    "python-dotenv>=1.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+    "pytest-cov>=4.1.0",
+    "httpx>=0.26.0",
+    "ruff>=0.2.0",
+    "mypy>=1.8.0",
+    "pre-commit>=3.6.0",
+]
+
+[project.scripts]
+scraperl = "app.main:run"
+
+[tool.hatch.build.targets.wheel]
+packages = ["app"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+addopts = "-v --tb=short"
+
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B", "C4", "SIM"]
+ignore = ["E501"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+warn_return_any = true
+warn_unused_ignores = true

backend/requirements.txt

@@ -0,0 +1,43 @@
+# Core Framework
+fastapi>=0.109.0
+uvicorn[standard]>=0.27.0
+pydantic>=2.5.0
+pydantic-settings>=2.1.0
+
+# HTTP Client
+httpx>=0.26.0
+
+# Vector Database
+chromadb>=0.4.22
+
+# HTML Processing
+beautifulsoup4>=4.12.0
+lxml>=5.1.0
+
+# Search Providers
+duckduckgo-search>=6.0.0
+
+# LLM Providers
+openai>=1.10.0
+anthropic>=0.18.0
+google-generativeai>=0.4.0
+groq>=0.4.0
+
+# Browser Automation
+playwright>=1.41.0
+
+# Tokenization
+tiktoken>=0.5.0
+
+# Utilities
+numpy>=1.26.0
+tenacity>=8.2.0
+structlog>=24.1.0
+python-dotenv>=1.0.0
+
+# Development
+pytest>=8.0.0
+pytest-asyncio>=0.23.0
+pytest-cov>=4.1.0
+ruff>=0.2.0
+mypy>=1.8.0