Update main.py

main.py

@@ -2,9 +2,8 @@ import httpx
 from fastapi import FastAPI, Request, HTTPException
 from starlette.responses import StreamingResponse, JSONResponse
 from starlette.background import BackgroundTask
-from pydantic_settings import BaseSettings
 from pydantic import Field, field_validator
-from typing import Set
+from typing import Set, Dict
 from contextlib import asynccontextmanager
 import os
 import random
@@ -15,6 +14,18 @@ import uuid
 from faker import Faker
 from fake_useragent import UserAgent
 
+# --- Pydantic V1/V2 Compatibility ---
+# This block makes the code resilient to different Pydantic versions.
+try:
+    # Recommended for Pydantic V2
+    from pydantic_settings import BaseSettings
+    print("Using pydantic_settings.BaseSettings")
+except ImportError:
+    # Fallback for Pydantic V1
+    from pydantic import BaseSettings
+    print("pydantic_settings not found, falling back to pydantic.BaseSettings")
+
+
 # --- Structured Configuration using Pydantic ---
 class Settings(BaseSettings):
     """Manages application settings and configuration from environment variables."""
@@ -22,11 +33,10 @@ class Settings(BaseSettings):
     TARGET_URL: str = Field(default="https://api.gmi-serving.com")
     MAX_RETRIES: int = Field(default=5, gt=0)
     
-    # Retry logic settings
     RETRY_CODES_STR: str = Field(default="429,500,502,503,504", alias="RETRY_CODES")
     RETRY_STATUS_CODES: Set[int] = {429, 500, 502, 503, 504}
-    BACKOFF_FACTOR: float = Field(default=0.5, gt=0) # Base delay for backoff in seconds
-    JITTER_FACTOR: float = Field(default=0.2, ge=0) # Randomness factor for backoff
+    BACKOFF_FACTOR: float = Field(default=0.5, gt=0)
+    JITTER_FACTOR: float = Field(default=0.2, ge=0)
 
     @field_validator('RETRY_STATUS_CODES', mode='before')
     @classmethod
@@ -45,22 +55,28 @@ class Settings(BaseSettings):
 
 # --- Initialize Settings and Global Services ---
 settings = Settings()
-logging.basicConfig(
-    level=settings.LOG_LEVEL.upper(),
-    format='%(asctime)s - %(levelname)s - [%(request_id)s] - %(message)s'
-)
-# Add a filter to include request_id in logs
+
+# Custom logging filter to inject request_id
 class RequestIdFilter(logging.Filter):
     def filter(self, record):
         record.request_id = getattr(record, 'request_id', 'main')
         return True
 
+logging.basicConfig(
+    level=settings.LOG_LEVEL.upper(),
+    format='%(asctime)s - %(levelname)s - [%(request_id)s] - %(message)s'
+)
 logger = logging.getLogger(__name__)
 logger.addFilter(RequestIdFilter())
 
 # Initialize data generation tools
 faker = Faker()
-ua = UserAgent()
+try:
+    ua = UserAgent()
+except Exception:
+    # Fallback if fake-useragent server is down
+    ua = None
+    logger.warning("Could not initialize UserAgent. A fallback will be used.")
 
 # --- HTTPX Client Lifecycle Management ---
 @asynccontextmanager
@@ -77,26 +93,58 @@ async def lifespan(app: FastAPI):
 app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
 
 # --- Helper Functions ---
-def prepare_forward_headers(incoming_headers: dict, client_host: str) -> dict:
-    """Prepares headers for the downstream request, adding spoofed IP and User-Agent."""
+def get_random_user_agent() -> str:
+    """Returns a random User-Agent, with a fallback."""
+    if ua:
+        try:
+            return ua.random
+        except Exception:
+            logger.warning("Failed to get random User-Agent, using fallback.")
+    # Fallback User-Agent
+    return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
+
+def prepare_forward_headers(incoming_headers: Dict, client_host: str) -> (Dict, str):
+    """
+    Prepares headers for the downstream request, adding a comprehensive set of
+    spoofed IP and a random User-Agent.
+    """
+    # Start with a clean slate of lower-cased headers
     forward_headers = {k.lower(): v for k, v in incoming_headers.items()}
     
-    # Remove headers that are specific to the incoming request's connection
-    forward_headers.pop("host", None)
-    forward_headers.pop("content-length", None)
+    # Remove headers that are specific to the incoming request's connection or could leak info
+    headers_to_remove = [
+        "host", "content-length", "x-forwarded-for", "x-real-ip", "forwarded",
+        "via", "x-client-ip", "x-forwarded-proto", "x-forwarded-host"
+    ]
+    for h in headers_to_remove:
+        forward_headers.pop(h, None)
     
-    # Generate realistic fake data
     spoofed_ip = faker.ipv4_public()
-    user_agent = ua.random
     
-    # Add/overwrite headers to obfuscate the origin
+    # Add a comprehensive set of headers to mask the origin
     override_headers = {
-        "user-agent": user_agent,
-        "x-forwarded-for": f"{spoofed_ip}, {client_host}",
+        # Standard headers
+        "x-forwarded-for": f"{spoofed_ip}, {faker.ipv4_public()}", # Append a fake proxy chain
         "x-real-ip": spoofed_ip,
+        
+        # RFC 7239 standard, more structured
+        "forwarded": f"for={spoofed_ip};proto=https",
+        
+        # Common non-standard headers
+        "x-client-ip": spoofed_ip,
         "x-originating-ip": spoofed_ip,
         "x-remote-ip": spoofed_ip,
-        "x-remote-addr": spoofed_ip
+        "x-remote-addr": spoofed_ip,
+        
+        # Cloudflare-specific headers
+        "cf-connecting-ip": spoofed_ip,
+        "true-client-ip": spoofed_ip,
+        
+        # Other proxy headers
+        "via": "1.1 google", # Fake a passthrough via a common service
+        
+        # Dynamic User-Agent
+        "user-agent": get_random_user_agent(),
     }
     forward_headers.update(override_headers)
     
@@ -134,7 +182,6 @@ async def reverse_proxy_handler(request: Request):
     last_exception = None
     for attempt in range(settings.MAX_RETRIES):
         if attempt > 0:
-            # Exponential backoff with jitter
             backoff_delay = settings.BACKOFF_FACTOR * (2 ** (attempt - 1))
             jitter = random.uniform(-settings.JITTER_FACTOR, settings.JITTER_FACTOR) * backoff_delay
             sleep_duration = max(0, backoff_delay + jitter)
@@ -150,7 +197,6 @@ async def reverse_proxy_handler(request: Request):
             )
             resp = await client.send(req, stream=True)
 
-            # If status is not in retry codes, or it's the last attempt, stream the response back
             if resp.status_code not in settings.RETRY_STATUS_CODES or attempt == settings.MAX_RETRIES - 1:
                 duration_ms = (time.monotonic() - start_time) * 1000
                 log_func = logger.info if resp.is_success else logger.warning
@@ -165,18 +211,15 @@ async def reverse_proxy_handler(request: Request):
                     background=BackgroundTask(resp.aclose),
                 )
             
-            # If we are going to retry, close the current response to free the connection
             await resp.aclose()
             last_exception = f"Last failed attempt returned status code {resp.status_code}"
 
         except httpx.RequestError as e:
             last_exception = e
             logger.warning(f"HTTPX RequestError on attempt {attempt + 1}: {e}", extra=log_extra)
-            # If it's the last attempt, we will fall through and raise a 502
             if attempt == settings.MAX_RETRIES - 1:
                 break
     
-    # This part is reached only if all retries fail
     duration_ms = (time.monotonic() - start_time) * 1000
     logger.critical(
         f"Failed permanently: {request.method} {url.path} after {settings.MAX_RETRIES} attempts. latency={duration_ms:.2f}ms. Last error: {last_exception}",