diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..ff4d58cd2240a93b890480f253416ecd88f88716 --- /dev/null +++ b/.env.example @@ -0,0 +1,44 @@ +# === DeepShield backend config example === + +# Server +APP_HOST=0.0.0.0 +APP_PORT=8000 +DEBUG=false +CORS_ORIGINS=["http://localhost:5173"] + +# === Database === +# SQLite (default — zero-config, great for dev / college demo): +DATABASE_URL=sqlite:///./deepshield.db +# Postgres (production path — run migrations are applied automatically +# by init_db via ALTER TABLE when new columns are missing): +# DATABASE_URL=postgresql+psycopg2://deepshield:CHANGEME@localhost:5432/deepshield + +# Phase 19.1 — SHA-256 dedup cache TTL (days) +CACHE_TTL_DAYS=30 + +# Phase 19.2 — object storage root (content-addressed media + thumbnails) +MEDIA_ROOT=./media + +# File upload +MAX_UPLOAD_SIZE_MB=100 +UPLOAD_DIR=./temp_uploads + +# AI models +PRELOAD_MODELS=true +DEVICE=cpu + +# LLM explainability (Phase 12) +LLM_PROVIDER=gemini +LLM_API_KEY= +LLM_MODEL=gemini-1.5-flash + +# News lookup (Phase 13) +NEWS_API_KEY= + +# Auth (REQUIRED in production — generate with python -c "import secrets; print(secrets.token_urlsafe(48))") +JWT_SECRET_KEY=change-me-in-production +JWT_ALGORITHM=HS256 +JWT_EXPIRATION_MINUTES=1440 + +# Optional metadata writer +EXIFTOOL_PATH= diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..f9eba9dca90eb44f013683bb618534bc50ae4f73 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +media/2f/2f7d41a5b57702a9a238409e6a1b973b4398f94c51fdf447e11782ed07693f06.jpg filter=lfs diff=lfs merge=lfs -text +media/63/635f21138244fc1dcbff5d0525b3c0a8187b1b9cc0ad90b5bb297a76e7b3850c.jpg filter=lfs diff=lfs merge=lfs -text +media/7b/7b626d0ddff59ca602e2e1eb02e62e21093aa647ab53c200ca5203f7fc17f6dd.jpg filter=lfs diff=lfs merge=lfs -text +media/c0/c064c839c9469d7b616db135f08e09235abd3d73f0889d978d1f92243226a028.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/Colab_ViT_Training.ipynb b/Colab_ViT_Training.ipynb deleted file mode 100644 index cb7dff448bd445cb9a01a51ca01984864890df14..0000000000000000000000000000000000000000 --- a/Colab_ViT_Training.ipynb +++ /dev/null @@ -1,233 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1e0e7b4a", - "metadata": {}, - "source": [ - "# DeepShield: FaceForensics++ ViT Training \n", - "Run this entirely in Google Colab.\n", - "**Before running**:\n", - "1. Go to `Runtime` -> `Change runtime type` -> select **T4 GPU**.\n", - "2. Run the cells below sequentially.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4fe293e7", - "metadata": {}, - "outputs": [], - "source": [ - "!pip install timm transformers datasets accelerate evaluate opencv-python\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c9387c0f", - "metadata": {}, - "outputs": [], - "source": [ - "# We create the download script inside the Colab environment\n", - "download_script = '''#!/usr/bin/env python\n", - "import argparse\n", - "import os\n", - "import urllib.request\n", - "import tempfile\n", - "import time\n", - "import sys\n", - "import json\n", - "from tqdm import tqdm\n", - "from os.path import join\n", - "\n", - "FILELIST_URL = 'misc/filelist.json'\n", - "DEEPFEAKES_DETECTION_URL = 'misc/deepfake_detection_filenames.json'\n", - "DEEPFAKES_MODEL_NAMES = ['decoder_A.h5', 'decoder_B.h5', 'encoder.h5',]\n", - "DATASETS = {\n", - " 'original': 'original_sequences/youtube',\n", - " 'Deepfakes': 'manipulated_sequences/Deepfakes',\n", - " 'Face2Face': 'manipulated_sequences/Face2Face',\n", - " 'FaceShifter': 'manipulated_sequences/FaceShifter',\n", - " 'FaceSwap': 'manipulated_sequences/FaceSwap',\n", - " 'NeuralTextures': 'manipulated_sequences/NeuralTextures'\n", - "}\n", - "ALL_DATASETS = ['original', 'Deepfakes', 'Face2Face', 'FaceShifter', 'FaceSwap', 'NeuralTextures']\n", - "COMPRESSION = ['raw', 'c23', 'c40']\n", - "TYPE = ['videos']\n", - "\n", - "def download_file(url, out_file):\n", - " os.makedirs(os.path.dirname(out_file), exist_ok=True)\n", - " if not os.path.isfile(out_file):\n", - " urllib.request.urlretrieve(url, out_file)\n", - "\n", - "def main():\n", - " parser = argparse.ArgumentParser()\n", - " parser.add_argument('output_path', type=str)\n", - " parser.add_argument('-d', '--dataset', type=str, default='all')\n", - " parser.add_argument('-c', '--compression', type=str, default='c40')\n", - " parser.add_argument('-t', '--type', type=str, default='videos')\n", - " parser.add_argument('-n', '--num_videos', type=int, default=50) # Small amount for tutorial\n", - " args = parser.parse_args()\n", - " \n", - " base_url = 'http://kaldir.vc.in.tum.de/faceforensics/v3/'\n", - " \n", - " datasets = [args.dataset] if args.dataset != 'all' else ALL_DATASETS\n", - " for dataset in datasets:\n", - " dataset_path = DATASETS[dataset]\n", - " print(f'Downloading {args.compression} of {dataset}')\n", - " \n", - " file_pairs = json.loads(urllib.request.urlopen(base_url + FILELIST_URL).read().decode(\"utf-8\"))\n", - " filelist = []\n", - " if 'original' in dataset_path:\n", - " for pair in file_pairs:\n", - " filelist += pair\n", - " else:\n", - " for pair in file_pairs:\n", - " filelist.append('_'.join(pair))\n", - " filelist.append('_'.join(pair[::-1]))\n", - " \n", - " filelist = filelist[:args.num_videos]\n", - " dataset_videos_url = base_url + f'{dataset_path}/{args.compression}/{args.type}/'\n", - " dataset_output_path = join(args.output_path, dataset_path, args.compression, args.type)\n", - " \n", - " for filename in tqdm(filelist):\n", - " download_file(dataset_videos_url + filename + \".mp4\", join(dataset_output_path, filename + \".mp4\"))\n", - "\n", - "if __name__ == \"__main__\":\n", - " main()\n", - "'''\n", - "\n", - "with open(\"download_ffpp.py\", \"w\") as f:\n", - " f.write(download_script)\n", - "\n", - "!python download_ffpp.py ./data -d all -c c40 -t videos -n 50\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f33716f6", - "metadata": {}, - "outputs": [], - "source": [ - "import cv2\n", - "import os\n", - "import glob\n", - "from tqdm import tqdm\n", - "\n", - "def extract_frames(video_folder, output_folder, label, max_frames=4):\n", - " os.makedirs(output_folder, exist_ok=True)\n", - " videos = glob.glob(os.path.join(video_folder, \"*.mp4\"))\n", - " \n", - " for vid_path in tqdm(videos, desc=f\"Extracting {label}\"):\n", - " vid_name = os.path.basename(vid_path).replace('.mp4','')\n", - " cap = cv2.VideoCapture(vid_path)\n", - " count = 0\n", - " while cap.isOpened() and count < max_frames:\n", - " ret, frame = cap.read()\n", - " if not ret: break\n", - " frame = cv2.resize(frame, (224, 224))\n", - " out_path = os.path.join(output_folder, f\"{vid_name}_f{count}.jpg\")\n", - " cv2.imwrite(out_path, frame)\n", - " count += 1\n", - " cap.release()\n", - "\n", - "# Extract Real\n", - "extract_frames('./data/original_sequences/youtube/c40/videos', './dataset/real', 'real')\n", - "\n", - "# Extract Fakes\n", - "fakes = ['Deepfakes', 'Face2Face', 'FaceSwap', 'NeuralTextures']\n", - "for f in fakes:\n", - " extract_frames(f'./data/manipulated_sequences/{f}/c40/videos', './dataset/fake', 'fake')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b79cdd85", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from datasets import load_dataset\n", - "from transformers import ViTImageProcessor, ViTForImageClassification, TrainingArguments, Trainer\n", - "import torch\n", - "\n", - "# 1. Load Dataset\n", - "dataset = load_dataset('imagefolder', data_dir='./dataset')\n", - "# Split into train/validation\n", - "dataset = dataset['train'].train_test_split(test_size=0.1)\n", - "\n", - "# 2. Preprocessor\n", - "model_name_or_path = 'google/vit-base-patch16-224-in21k'\n", - "processor = ViTImageProcessor.from_pretrained(model_name_or_path)\n", - "\n", - "def transform(example_batch):\n", - " # Take a list of PIL images and turn them to pixel values\n", - " inputs = processor([x.convert(\"RGB\") for x in example_batch['image']], return_tensors='pt')\n", - " inputs['labels'] = example_batch['label']\n", - " return inputs\n", - "\n", - "prepared_ds = dataset.with_transform(transform)\n", - "\n", - "def collate_fn(batch):\n", - " return {\n", - " 'pixel_values': torch.stack([x['pixel_values'] for x in batch]),\n", - " 'labels': torch.tensor([x['labels'] for x in batch])\n", - " }\n", - "\n", - "# 3. Load Model\n", - "labels = dataset['train'].features['label'].names\n", - "model = ViTForImageClassification.from_pretrained(\n", - " model_name_or_path,\n", - " num_labels=len(labels),\n", - " id2label={str(i): c for i, c in enumerate(labels)},\n", - " label2id={c: str(i) for i, c in enumerate(labels)}\n", - ")\n", - "\n", - "training_args = TrainingArguments(\n", - " output_dir=\"./vit-deepshield\",\n", - " per_device_train_batch_size=16,\n", - " eval_strategy=\"steps\",\n", - " num_train_epochs=3,\n", - " fp16=True, # Mixed precision for speed\n", - " save_steps=100,\n", - " eval_steps=100,\n", - " logging_steps=10,\n", - " learning_rate=2e-4,\n", - " save_total_limit=2,\n", - " remove_unused_columns=False,\n", - " push_to_hub=False,\n", - " load_best_model_at_end=True,\n", - ")\n", - "\n", - "import evaluate\n", - "metric = evaluate.load(\"accuracy\")\n", - "def compute_metrics(p):\n", - " return metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids)\n", - "\n", - "trainer = Trainer(\n", - " model=model,\n", - " args=training_args,\n", - " data_collator=collate_fn,\n", - " compute_metrics=compute_metrics,\n", - " train_dataset=prepared_ds[\"train\"],\n", - " eval_dataset=prepared_ds[\"test\"],\n", - ")\n", - "\n", - "# 4. Train\n", - "train_results = trainer.train()\n", - "trainer.save_model(\"deepshield_vit_model\")\n", - "processor.save_pretrained(\"deepshield_vit_model\")\n", - "trainer.log_metrics(\"train\", train_results.metrics)\n", - "trainer.save_metrics(\"train\", train_results.metrics)\n", - "trainer.save_state()\n", - "print(\"Training Complete! The model is saved to ./deepshield_vit_model\")\n" - ] - } - ], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/README.md b/README.md deleted file mode 100644 index bf0aca7cb4f5680b6a0f7f9d7471e859777d3923..0000000000000000000000000000000000000000 --- a/README.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Deepshield -emoji: 🛡️ -colorFrom: blue -colorTo: indigo -sdk: docker -app_port: 7860 -pinned: true ---- - -# DeepShield Backend - -This space hosts the FastAPI backend for DeepShield. diff --git a/analyze.py b/analyze.py deleted file mode 100644 index 252939f3c6a78c228a52137684a7511b0692426c..0000000000000000000000000000000000000000 --- a/analyze.py +++ /dev/null @@ -1,177 +0,0 @@ -from __future__ import annotations - -from typing import List - -from pydantic import BaseModel - -from schemas.common import ( - ArtifactIndicator, - ContradictingEvidence, - ExifSummary, - LLMExplainabilitySummary, - ProcessingSummary, - TrustedSource, - TruthOverride, - Verdict, - VLMBreakdown, -) - - -class SensationalismBreakdown(BaseModel): - score: int = 0 - level: str = "Low" - exclamation_count: int = 0 - caps_word_count: int = 0 - clickbait_matches: int = 0 - emotional_word_count: int = 0 - superlative_count: int = 0 - - -class ManipulationIndicatorOut(BaseModel): - pattern_type: str - matched_text: str - start_pos: int - end_pos: int - severity: str - description: str - - -class TextExplainability(BaseModel): - fake_probability: float - top_label: str - all_scores: dict = {} - keywords: List[str] = [] - sensationalism: SensationalismBreakdown = SensationalismBreakdown() - manipulation_indicators: List[ManipulationIndicatorOut] = [] - detected_language: str = "en" # ISO 639-1 code, e.g. "en", "hi" - truth_override: TruthOverride | None = None - - -class TextAnalysisResponse(BaseModel): - analysis_id: str - record_id: int = 0 - media_type: str = "text" - timestamp: str - verdict: Verdict - explainability: TextExplainability - llm_summary: LLMExplainabilitySummary | None = None - trusted_sources: List[TrustedSource] = [] - contradicting_evidence: List[ContradictingEvidence] = [] - processing_summary: ProcessingSummary - responsible_ai_notice: str = ( - "AI-based analysis may not be 100% accurate. Cross-check with trusted sources before sharing." - ) - - -class OCRBoxOut(BaseModel): - text: str - bbox: List[List[int]] - confidence: float - - -class SuspiciousPhraseOut(BaseModel): - text: str - bbox: List[List[int]] - pattern_type: str - severity: str - description: str - - -class LayoutAnomalyOut(BaseModel): - type: str - severity: str - description: str - confidence: float - - -class ScreenshotExplainability(BaseModel): - extracted_text: str = "" - ocr_boxes: List[OCRBoxOut] = [] - fake_probability: float = 0.0 - sensationalism: SensationalismBreakdown = SensationalismBreakdown() - suspicious_phrases: List[SuspiciousPhraseOut] = [] - layout_anomalies: List[LayoutAnomalyOut] = [] - keywords: List[str] = [] - detected_language: str = "en" - truth_override: TruthOverride | None = None - - -class ScreenshotAnalysisResponse(BaseModel): - analysis_id: str - record_id: int = 0 - media_type: str = "screenshot" - timestamp: str - verdict: Verdict - explainability: ScreenshotExplainability - llm_summary: LLMExplainabilitySummary | None = None - trusted_sources: List[TrustedSource] = [] - contradicting_evidence: List[ContradictingEvidence] = [] - processing_summary: ProcessingSummary - responsible_ai_notice: str = ( - "AI-based analysis may not be 100% accurate. Cross-check with trusted sources before sharing." - ) - - -class ImageExplainability(BaseModel): - heatmap_base64: str = "" - ela_base64: str = "" - boxes_base64: str = "" - heatmap_status: str = "success" # success | failed | degraded - artifact_indicators: List[ArtifactIndicator] = [] - exif: ExifSummary | None = None - llm_summary: LLMExplainabilitySummary | None = None - vlm_breakdown: VLMBreakdown | None = None - - -class FrameAnalysisOut(BaseModel): - index: int - timestamp_s: float - label: str - confidence: float - suspicious_prob: float - is_suspicious: bool - has_face: bool = False - scored: bool = False - - -class VideoExplainability(BaseModel): - num_frames_sampled: int - num_face_frames: int = 0 - num_suspicious_frames: int - mean_suspicious_prob: float - max_suspicious_prob: float - suspicious_ratio: float - insufficient_faces: bool = False - suspicious_timestamps: List[float] = [] - frames: List[FrameAnalysisOut] = [] - - -class VideoAnalysisResponse(BaseModel): - analysis_id: str - record_id: int = 0 - media_type: str = "video" - timestamp: str - verdict: Verdict - explainability: VideoExplainability - llm_summary: LLMExplainabilitySummary | None = None - trusted_sources: List[TrustedSource] = [] - contradicting_evidence: List[ContradictingEvidence] = [] - processing_summary: ProcessingSummary - responsible_ai_notice: str = ( - "AI-based analysis may not be 100% accurate. Cross-check with trusted sources before sharing." - ) - - -class ImageAnalysisResponse(BaseModel): - analysis_id: str - record_id: int = 0 - media_type: str = "image" - timestamp: str - verdict: Verdict - explainability: ImageExplainability - trusted_sources: List[TrustedSource] = [] - contradicting_evidence: List[ContradictingEvidence] = [] - processing_summary: ProcessingSummary - responsible_ai_notice: str = ( - "AI-based analysis may not be 100% accurate. Cross-check with trusted sources before sharing." - ) diff --git a/api/router.py b/api/router.py index 478bb749be5a898755712262b76c49db8bd1257f..41a659ed905de4a596aaa04e18b27bf721cc7a89 100644 --- a/api/router.py +++ b/api/router.py @@ -5,6 +5,7 @@ from api.v1 import analyze, auth, health, history, report api_router = APIRouter(prefix="/api/v1") api_router.include_router(health.router) api_router.include_router(analyze.router) +api_router.include_router(analyze.jobs_router) # Phase 19.3 api_router.include_router(report.router) api_router.include_router(auth.router) api_router.include_router(history.router) diff --git a/api/v1/health.py b/api/v1/health.py index b02fd9845b16997bda02ffe00e91915c4c043533..b67239da277a8704702dc4bc08917e031201db57 100644 --- a/api/v1/health.py +++ b/api/v1/health.py @@ -1,8 +1,79 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Response, status +from loguru import logger +from sqlalchemy import text + +from config import settings +from db.database import engine +from services.llm_explainer import is_rate_limited router = APIRouter(tags=["health"]) @router.get("/health") def health(): + """Legacy combined healthcheck — kept for backwards compatibility.""" return {"status": "ok", "service": "deepshield-backend"} + + +@router.get("/health/live") +def health_live(): + """Liveness probe — returns 200 as long as the process is up.""" + return {"status": "alive"} + + +@router.get("/health/ready") +def health_ready(response: Response): + """Readiness probe — 200 only when DB is reachable and models are loaded. + + Phase 19.5: the frontend disables the Analyze button while this returns 503. + """ + checks: dict[str, bool] = {} + + try: + with engine.connect() as conn: + conn.execute(text("SELECT 1")) + checks["db"] = True + except Exception as e: # noqa: BLE001 + logger.warning(f"readiness db check failed: {e}") + checks["db"] = False + + try: + from models.model_loader import get_model_loader + checks["models"] = bool(get_model_loader().is_ready()) + except AttributeError: + # No is_ready() — fall back to "ready if loader constructs" + try: + from models.model_loader import get_model_loader + get_model_loader() + checks["models"] = True + except Exception: # noqa: BLE001 + checks["models"] = False + except Exception as e: # noqa: BLE001 + logger.warning(f"readiness model check failed: {e}") + checks["models"] = False + + ok = all(checks.values()) + if not ok: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return {"status": "ready" if ok else "not_ready", "checks": checks} + + +@router.get("/health/llm") +def health_llm(response: Response): + """LLM availability probe — lets the frontend decide whether to request/show + the AI summary card. Doesn't spend tokens; only checks config + breaker state. + """ + has_primary = bool(settings.LLM_API_KEY) + has_fallback = bool(settings.GROQ_API_KEY) + cooldown = is_rate_limited() + + # Available if (any provider configured) AND (not rate-limited OR fallback exists) + available = (has_primary or has_fallback) and (not cooldown or has_fallback) + if not available: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return { + "available": available, + "primary": f"{settings.LLM_PROVIDER}/{settings.LLM_MODEL}" if has_primary else None, + "fallback": f"groq/{settings.GROQ_MODEL}" if has_fallback else None, + "rate_limited": cooldown, + } diff --git a/api/v1/history.py b/api/v1/history.py index db70c77e068a5e4f8070caddc011504868912493..40e53ef8a4a8e358d387d604ae217ed57e385a1d 100644 --- a/api/v1/history.py +++ b/api/v1/history.py @@ -60,7 +60,13 @@ def get_history_detail( if not r or r.user_id != user.id: raise HTTPException(status.HTTP_404_NOT_FOUND, "Analysis not found") try: - return json.loads(r.result_json) + payload = json.loads(r.result_json) + # Inject storage fields from DB columns so the frontend can display full-size media + if r.media_path and not payload.get("media_path"): + payload["media_path"] = r.media_path + if r.thumbnail_url and not payload.get("thumbnail_url"): + payload["thumbnail_url"] = r.thumbnail_url + return payload except Exception: raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Corrupt result payload") diff --git a/api/v1/report.py b/api/v1/report.py index 72a34c8165dbd78f8e474afdc6d9df77d6e54494..985acdd27fd218c1b6af36b994b3c94d4e80ea9a 100644 --- a/api/v1/report.py +++ b/api/v1/report.py @@ -2,24 +2,45 @@ from __future__ import annotations from pathlib import Path -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.responses import FileResponse from loguru import logger from sqlalchemy.orm import Session +from api.deps import get_current_user, optional_current_user from db.database import get_db -from db.models import AnalysisRecord, Report +from db.models import AnalysisRecord, Report, User +from services.rate_limit import ANON_REPORT, AUTH_REPORT, is_anon, is_authed, limiter from services.report_service import cleanup_expired, create_report_row, generate_report router = APIRouter(prefix="/report", tags=["report"]) +def _assert_record_access(record: AnalysisRecord, user: User | None) -> None: + """Phase 15.1 — allow access if the requester owns the record, or if the record + is anonymous (user_id is None). Everything else is 403.""" + if record.user_id is None: + return + if user is not None and record.user_id == user.id: + return + raise HTTPException(status.HTTP_403_FORBIDDEN, "You do not own this analysis") + + @router.post("/{analysis_id}") -def generate(analysis_id: int, db: Session = Depends(get_db)): +@limiter.limit(ANON_REPORT, exempt_when=is_authed) +@limiter.limit(AUTH_REPORT, exempt_when=is_anon) +def generate( + request: Request, + analysis_id: int, + db: Session = Depends(get_db), + user: User | None = Depends(optional_current_user), +): record = db.query(AnalysisRecord).filter(AnalysisRecord.id == analysis_id).first() if not record: raise HTTPException(status_code=404, detail="analysis not found") + _assert_record_access(record, user) + existing = db.query(Report).filter(Report.analysis_id == analysis_id).first() if existing and Path(existing.file_path).exists(): return {"report_id": existing.id, "analysis_id": analysis_id, "ready": True} @@ -44,7 +65,19 @@ def generate(analysis_id: int, db: Session = Depends(get_db)): @router.get("/{analysis_id}/download") -def download(analysis_id: int, db: Session = Depends(get_db)): +@limiter.limit(ANON_REPORT, exempt_when=is_authed) +@limiter.limit(AUTH_REPORT, exempt_when=is_anon) +def download( + request: Request, + analysis_id: int, + db: Session = Depends(get_db), + user: User | None = Depends(optional_current_user), +): + record = db.query(AnalysisRecord).filter(AnalysisRecord.id == analysis_id).first() + if not record: + raise HTTPException(status_code=404, detail="analysis not found") + _assert_record_access(record, user) + row = db.query(Report).filter(Report.analysis_id == analysis_id).first() if not row: raise HTTPException(status_code=404, detail="report not found — generate first") @@ -58,7 +91,9 @@ def download(analysis_id: int, db: Session = Depends(get_db)): ) -@router.post("/cleanup") -def cleanup(): +@router.post("/cleanup", include_in_schema=False) +def cleanup(user: User = Depends(get_current_user)): + # Phase 15.1 — auth-guarded. Exposed only to authenticated users; an internal + # scheduler loop in main.py handles periodic cleanup automatically. n = cleanup_expired() return {"deleted": n} diff --git a/artifact_detector.py b/artifact_detector.py deleted file mode 100644 index bb05435afa5d80def094f961af0390679659358e..0000000000000000000000000000000000000000 --- a/artifact_detector.py +++ /dev/null @@ -1,229 +0,0 @@ -from __future__ import annotations - -import io -from typing import List - -import numpy as np -from loguru import logger -from PIL import Image - -from schemas.common import ArtifactIndicator - - -def _severity_from_score(score: float) -> str: - if score >= 0.7: - return "high" - if score >= 0.4: - return "medium" - return "low" - - -# ---------- 1. GAN high-frequency signature (FFT) ---------- -def detect_gan_hf_artifact(pil_img: Image.Image) -> ArtifactIndicator | None: - """Compute high-frequency energy ratio on the luminance channel. - Real photos typically follow a ~1/f spectrum; many GAN outputs show - elevated HF energy or spectral peaks. - """ - try: - gray = np.asarray(pil_img.convert("L"), dtype=np.float32) - # downsample for speed - if max(gray.shape) > 512: - import cv2 - - scale = 512 / max(gray.shape) - gray = cv2.resize(gray, (int(gray.shape[1] * scale), int(gray.shape[0] * scale))) - - fft = np.fft.fftshift(np.fft.fft2(gray)) - mag = np.abs(fft) - h, w = mag.shape - cy, cx = h // 2, w // 2 - y, x = np.ogrid[:h, :w] - r = np.sqrt((x - cx) ** 2 + (y - cy) ** 2) - r_max = np.sqrt(cx * cx + cy * cy) - hf_mask = r > (0.5 * r_max) - - total = float(mag.sum() + 1e-9) - hf = float(mag[hf_mask].sum()) - ratio = hf / total # typically 0.05–0.20 for natural photos - - # normalize to [0,1] suspiciousness - score = max(0.0, min(1.0, (ratio - 0.10) / 0.20)) - sev = _severity_from_score(score) - return ArtifactIndicator( - type="gan_artifact", - severity=sev, - description=( - f"High-frequency energy ratio {ratio:.3f} — " - + ("elevated HF energy consistent with GAN/diffusion outputs" if score > 0.4 - else "natural frequency falloff") - ), - confidence=float(score), - ) - except Exception as e: # noqa: BLE001 - logger.warning(f"GAN HF detection failed: {e}") - return None - - -# ---------- 2. JPEG quantization table anomaly ---------- -_STANDARD_Q_SUMS = { # rough heuristic: camera JPEGs fall in these ranges - 50: (1500, 4500), - 75: (600, 2500), - 90: (200, 1000), - 95: (100, 600), -} - - -def detect_compression_anomaly(raw_bytes: bytes) -> ArtifactIndicator | None: - """Inspect JPEG quantization tables. Missing tables, non-standard layouts, - or re-saved tables often indicate manipulation or re-encoding. - """ - try: - img = Image.open(io.BytesIO(raw_bytes)) - if img.format != "JPEG": - return ArtifactIndicator( - type="compression", - severity="low", - description=f"Non-JPEG format ({img.format}); compression signature not available", - confidence=0.1, - ) - - q = getattr(img, "quantization", None) - if not q: - return ArtifactIndicator( - type="compression", - severity="low", - description="No JPEG quantization tables readable", - confidence=0.2, - ) - - tables = list(q.values()) - sums = [int(sum(t)) for t in tables] - num_tables = len(tables) - - # Heuristics: very low sum → very high quality (possibly re-saved); - # non-standard number of tables; extreme values. - suspicious = 0.0 - reasons: list[str] = [] - if num_tables not in (1, 2): - suspicious += 0.4 - reasons.append(f"unusual table count ({num_tables})") - if any(s < 60 for s in sums): - suspicious += 0.3 - reasons.append("very low quantization sums (possible re-encoding)") - if any(s > 8000 for s in sums): - suspicious += 0.2 - reasons.append("very high quantization sums") - - score = max(0.0, min(1.0, suspicious)) - sev = _severity_from_score(score) - desc = ( - f"JPEG Q-table sums {sums}" - + (f"; {', '.join(reasons)}" if reasons else "; within typical camera range") - ) - return ArtifactIndicator( - type="compression", - severity=sev, - description=desc, - confidence=float(score), - ) - except Exception as e: # noqa: BLE001 - logger.warning(f"Compression anomaly detection failed: {e}") - return None - - -# ---------- 3. Facial boundary + 4. Lighting (MediaPipe) ---------- -def detect_face_based_artifacts(pil_img: Image.Image) -> List[ArtifactIndicator]: - """If a face is detected, analyze jaw boundary variance and per-quadrant - luminance balance. Returns 0, 1, or 2 indicators. - """ - results: List[ArtifactIndicator] = [] - try: - import mediapipe as mp # type: ignore - - from models.model_loader import get_model_loader - - detector = get_model_loader().load_face_detector() - rgb = np.asarray(pil_img.convert("RGB")) - h, w = rgb.shape[:2] - mp_result = detector.process(rgb) - - if not mp_result.multi_face_landmarks: - return results - - landmarks = mp_result.multi_face_landmarks[0].landmark - - # ----- Jaw boundary jitter ----- - # FaceMesh jaw/oval landmark indices (approximate face contour) - JAW_IDX = [ - 10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, - 288, 397, 365, 379, 378, 400, 377, 152, 148, 176, 149, - 150, 136, 172, 58, 132, 93, 234, 127, 162, 21, 54, 103, 67, 109, - ] - pts = np.array([(landmarks[i].x * w, landmarks[i].y * h) for i in JAW_IDX]) - # Second-difference magnitude = local curvature jitter - diffs = np.diff(pts, axis=0) - seconds = np.diff(diffs, axis=0) - jitter = float(np.linalg.norm(seconds, axis=1).mean()) / max(w, h) - jitter_score = max(0.0, min(1.0, (jitter - 0.003) / 0.010)) - results.append( - ArtifactIndicator( - type="facial_boundary", - severity=_severity_from_score(jitter_score), - description=( - f"Jaw-contour jitter {jitter:.4f} (normalized) — " - + ("inconsistent boundary blending detected" if jitter_score > 0.4 - else "face boundary appears smooth") - ), - confidence=float(jitter_score), - ) - ) - - # ----- Lighting inconsistency (per-quadrant luminance) ----- - xs = np.array([lm.x * w for lm in landmarks]) - ys = np.array([lm.y * h for lm in landmarks]) - x0, x1 = int(max(0, xs.min())), int(min(w, xs.max())) - y0, y1 = int(max(0, ys.min())), int(min(h, ys.max())) - if x1 > x0 + 4 and y1 > y0 + 4: - face_crop = rgb[y0:y1, x0:x1] - gray = 0.299 * face_crop[..., 0] + 0.587 * face_crop[..., 1] + 0.114 * face_crop[..., 2] - hh, ww = gray.shape - quads = [ - gray[: hh // 2, : ww // 2], - gray[: hh // 2, ww // 2 :], - gray[hh // 2 :, : ww // 2], - gray[hh // 2 :, ww // 2 :], - ] - means = np.array([q.mean() for q in quads if q.size > 0]) - if means.size == 4 and means.mean() > 1e-3: - imbalance = float(means.std() / means.mean()) - lighting_score = max(0.0, min(1.0, (imbalance - 0.08) / 0.20)) - results.append( - ArtifactIndicator( - type="lighting", - severity=_severity_from_score(lighting_score), - description=( - f"Luminance imbalance across face quadrants {imbalance:.3f} — " - + ("inconsistent lighting direction" if lighting_score > 0.4 - else "lighting appears uniform") - ), - confidence=float(lighting_score), - ) - ) - except Exception as e: # noqa: BLE001 - logger.warning(f"Face-based artifact detection failed: {e}") - - return results - - -# ---------- Orchestrator ---------- -def scan_artifacts(pil_img: Image.Image, raw_bytes: bytes) -> List[ArtifactIndicator]: - indicators: List[ArtifactIndicator] = [] - for fn in ( - lambda: detect_gan_hf_artifact(pil_img), - lambda: detect_compression_anomaly(raw_bytes), - ): - ind = fn() - if ind is not None: - indicators.append(ind) - indicators.extend(detect_face_based_artifacts(pil_img)) - return indicators diff --git a/auth.py b/auth.py deleted file mode 100644 index ed5ddb27c8400f6bcf3556bc85e662ff28295fc1..0000000000000000000000000000000000000000 --- a/auth.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from datetime import datetime - -from pydantic import BaseModel, EmailStr, Field - - -class RegisterBody(BaseModel): - email: EmailStr - password: str = Field(min_length=6, max_length=128) - name: str | None = Field(default=None, max_length=255) - - -class LoginBody(BaseModel): - email: EmailStr - password: str - - -class UserOut(BaseModel): - id: int - email: str - name: str | None = None - created_at: datetime - - -class TokenResponse(BaseModel): - access_token: str - token_type: str = "bearer" - expires_in_minutes: int - user: UserOut diff --git a/auth_service.py b/auth_service.py deleted file mode 100644 index 2c63225dc14fa4b8f1c06f155b0e9386d34f6a91..0000000000000000000000000000000000000000 --- a/auth_service.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta, timezone -from typing import Any - -import bcrypt -from jose import JWTError, jwt -from sqlalchemy.orm import Session - -from config import settings -from db.models import User - - -def _encode_pw(plain: str) -> bytes: - # bcrypt truncates to 72 bytes silently in some builds and hard-errors in others. - # Truncate explicitly so behavior is deterministic across versions. - return plain.encode("utf-8")[:72] - - -def hash_password(plain: str) -> str: - return bcrypt.hashpw(_encode_pw(plain), bcrypt.gensalt()).decode("utf-8") - - -def verify_password(plain: str, hashed: str) -> bool: - try: - return bcrypt.checkpw(_encode_pw(plain), hashed.encode("utf-8")) - except Exception: - return False - - -def create_access_token(user_id: int, email: str) -> str: - now = datetime.now(timezone.utc) - payload = { - "sub": str(user_id), - "email": email, - "iat": int(now.timestamp()), - "exp": int((now + timedelta(minutes=settings.JWT_EXPIRATION_MINUTES)).timestamp()), - } - return jwt.encode(payload, settings.JWT_SECRET_KEY, algorithm=settings.JWT_ALGORITHM) - - -def decode_token(token: str) -> dict[str, Any] | None: - try: - return jwt.decode(token, settings.JWT_SECRET_KEY, algorithms=[settings.JWT_ALGORITHM]) - except JWTError: - return None - - -def register_user(db: Session, email: str, password: str, name: str | None) -> User: - email = email.strip().lower() - user = User(email=email, password_hash=hash_password(password), name=(name or None)) - db.add(user) - db.commit() - db.refresh(user) - return user - - -def authenticate(db: Session, email: str, password: str) -> User | None: - email = email.strip().lower() - user = db.query(User).filter(User.email == email).first() - if not user or not verify_password(password, user.password_hash): - return None - return user - - -def get_user(db: Session, user_id: int) -> User | None: - return db.query(User).filter(User.id == user_id).first() diff --git a/common.py b/common.py deleted file mode 100644 index 612d6f1f148744d26562707e935b98a287cb5448..0000000000000000000000000000000000000000 --- a/common.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -from typing import List, Optional - -from pydantic import BaseModel, ConfigDict, Field - - -class Verdict(BaseModel): - model_config = ConfigDict(protected_namespaces=()) - - label: str - severity: str - authenticity_score: int = Field(ge=0, le=100) - model_confidence: float = Field(ge=0.0, le=1.0) - model_label: str - - -class ArtifactIndicator(BaseModel): - type: str - severity: str # low | medium | high - description: str - confidence: float = Field(ge=0.0, le=1.0) - - -class TrustedSource(BaseModel): - source_name: str - title: str - url: str - published_at: Optional[str] = None - relevance_score: float = Field(ge=0.0, le=1.0) - - -class ContradictingEvidence(BaseModel): - source_name: str - title: str - url: str - type: str = "fact_check" - - -class TruthOverride(BaseModel): - applied: bool = False - source_url: str = "" - source_name: str = "" - similarity: float = 0.0 - fake_prob_before: float = 0.0 - fake_prob_after: float = 0.0 - - -class ExifSummary(BaseModel): - make: Optional[str] = None - model: Optional[str] = None - datetime_original: Optional[str] = None - gps_info: Optional[str] = None - software: Optional[str] = None - lens_model: Optional[str] = None - trust_adjustment: int = 0 # negative = more real, positive = more fake - trust_reason: str = "" - - -class LLMExplainabilitySummary(BaseModel): - paragraph: str = "" - bullets: List[str] = [] - model_used: str = "" - cached: bool = False - - -class VLMComponentScore(BaseModel): - score: int = Field(ge=0, le=100, default=75) - notes: str = "" - - -class VLMBreakdown(BaseModel): - facial_symmetry: VLMComponentScore = VLMComponentScore() - skin_texture: VLMComponentScore = VLMComponentScore() - lighting_consistency: VLMComponentScore = VLMComponentScore() - background_coherence: VLMComponentScore = VLMComponentScore() - anatomy_hands_eyes: VLMComponentScore = VLMComponentScore() - context_objects: VLMComponentScore = VLMComponentScore() - model_used: str = "" - cached: bool = False - - -class ProcessingSummary(BaseModel): - model_config = ConfigDict(protected_namespaces=()) - - stages_completed: List[str] - total_duration_ms: int - model_used: str diff --git a/config.py b/config.py index f229070b32f8795926f40561ac2d65965fe437d1..a3b93cd825567cf4eaadf47dc4e15d327d9771ac 100644 --- a/config.py +++ b/config.py @@ -20,6 +20,7 @@ class Settings(BaseSettings): # AI Models IMAGE_MODEL_ID: str = "prithivMLmods/Deep-Fake-Detector-v2-Model" + GENERAL_IMAGE_MODEL_ID: str = "umm-maybe/AI-image-detector" TEXT_MODEL_ID: str = "jy46604790/Fake-News-Bert-Detect" # Multilingual text model for non-English (Hindi etc.). Leave empty to fall back to TEXT_MODEL_ID. TEXT_MULTILANG_MODEL_ID: str = "" @@ -37,15 +38,47 @@ class Settings(BaseSettings): REPORT_DIR: str = "./temp_reports" REPORT_TTL_SECONDS: int = 3600 # 1h expiry + # Phase 19 — dedup cache + object storage + CACHE_TTL_DAYS: int = 30 + MEDIA_ROOT: str = "./media" + # LLM Explainability (Phase 12) LLM_PROVIDER: str = "gemini" # "gemini" | "openai" LLM_API_KEY: str = "" - LLM_MODEL: str = "gemini-2.5-pro" # or "gpt-4o" + LLM_MODEL: str = "gemini-2.5-flash" # flash is ~12x cheaper + larger free-tier quota than pro. Use "gemini-2.5-pro" for harder reasoning. + + # LLM fallback — Groq (Llama 3.3 70B by default). Used automatically when the + # primary provider returns 429/quota exceeded. Leave empty to disable fallback. + GROQ_API_KEY: str = "" + GROQ_MODEL: str = "llama-3.3-70b-versatile" # EfficientNet (ICPR2020 / DeepShield1 merge) EFFICIENTNET_MODEL: str = "EfficientNetAutoAttB4" EFFICIENTNET_TRAIN_DB: str = "DFDC" ENSEMBLE_MODE: bool = True # run both ViT + EfficientNet and average scores + + # Phase 11.3: FFPP-fine-tuned ViT. Path is resolved relative to the repo root. + # The checkpoint lives at /trained_models/ (the `trained_models/` dir + # at the project root, alongside `backend/` and `frontend/`). + FFPP_MODEL_PATH: str = "trained_models" + # Optional: pull FFPP checkpoint from Hugging Face Hub when local checkpoint + # is missing (keeps large model files out of GitHub source repo). + FFPP_MODEL_REPO_ID: str = "" + FFPP_MODEL_REVISION: str = "main" + FFPP_BASE_PROCESSOR_ID: str = "google/vit-base-patch16-224-in21k" + FFPP_ENABLED: bool = True + # Ensemble weights — FFPP is trained on a better (face-specific FFPP c40) dataset + # and is weighted more heavily when a face is present. When no face is detected, + # we still blend it but lean on the generic ViT since FFPP only saw face crops. + FFPP_WEIGHT_FACE: float = 0.55 # face-present ensemble weight + VIT_WEIGHT_FACE: float = 0.20 + EFFNET_WEIGHT_FACE: float = 0.25 + FFPP_WEIGHT_NOFACE: float = 0.35 # no-face ensemble weight + VIT_WEIGHT_NOFACE: float = 0.65 + NOFACE_GENERAL_WEIGHT: float = 0.60 + NOFACE_FORENSICS_WEIGHT: float = 0.20 + NOFACE_EXIF_WEIGHT: float = 0.10 + NOFACE_VLM_WEIGHT: float = 0.10 VIDEO_SAMPLE_FRAMES: int = 16 # frames to sample per video for inference EXIFTOOL_PATH: str = "" # full path to ExifTool binary; empty = metadata write disabled diff --git a/database.py b/database.py deleted file mode 100644 index 17f8fef1a73af080cf4097a4929399046a3bfc4d..0000000000000000000000000000000000000000 --- a/database.py +++ /dev/null @@ -1,30 +0,0 @@ -from sqlalchemy import create_engine -from sqlalchemy.orm import DeclarativeBase, sessionmaker - -from config import settings - -engine = create_engine( - settings.DATABASE_URL, - connect_args={"check_same_thread": False} if settings.DATABASE_URL.startswith("sqlite") else {}, - pool_pre_ping=True, - pool_recycle=300, -) - -SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - - -class Base(DeclarativeBase): - pass - - -def get_db(): - db = SessionLocal() - try: - yield db - finally: - db.close() - - -def init_db(): - from db import models # noqa: F401 - Base.metadata.create_all(bind=engine) diff --git a/datasets/__init__.py b/datasets/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/datasets/build_manifest.py b/datasets/build_manifest.py deleted file mode 100644 index 00885270483c63cde5aafaffff604f70039084ae..0000000000000000000000000000000000000000 --- a/datasets/build_manifest.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Build a unified train/val/test manifest (70/15/15) across all dataset buckets. - -Expected input layout (produced by the other scripts in this package): - - data_root/ - real/ - ffpp_youtube/*.jpg # frames from FFPP original_sequences - ffhq/*.jpg # FFHQ thumbnails - - fake/ - ffpp_deepfakes/*.jpg - ffpp_face2face/*.jpg - ffpp_faceswap/*.jpg - ffpp_neuraltextures/*.jpg - ffpp_faceshifter/*.jpg - dfdc/*.jpg - -The manifest is stratified by (label, source) so FFHQ stays represented -in val/test. - -Usage: - python -m backend.training.datasets.build_manifest \ - --data ./data --out ./data/manifest.csv --seed 42 -""" -from __future__ import annotations - -import argparse -import csv -import random -from collections import defaultdict -from pathlib import Path - -IMG_EXTS = {".jpg", ".jpeg", ".png"} - - -def collect(data_root: Path) -> list[tuple[str, str, str]]: - rows: list[tuple[str, str, str]] = [] - for label in ("real", "fake"): - label_root = data_root / label - if not label_root.exists(): - continue - for source_dir in sorted(p for p in label_root.iterdir() if p.is_dir()): - for img in source_dir.rglob("*"): - if img.suffix.lower() in IMG_EXTS and img.is_file(): - rows.append((str(img.resolve()), label, source_dir.name)) - return rows - - -def split(rows: list[tuple[str, str, str]], seed: int) -> dict[str, list[tuple[str, str, str]]]: - buckets: dict[tuple[str, str], list[tuple[str, str, str]]] = defaultdict(list) - for r in rows: - buckets[(r[1], r[2])].append(r) - - rng = random.Random(seed) - out = {"train": [], "val": [], "test": []} - for key, items in buckets.items(): - rng.shuffle(items) - n = len(items) - n_train = int(0.70 * n) - n_val = int(0.15 * n) - out["train"].extend(items[:n_train]) - out["val"].extend(items[n_train : n_train + n_val]) - out["test"].extend(items[n_train + n_val :]) - return out - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--data", required=True, type=Path) - ap.add_argument("--out", required=True, type=Path) - ap.add_argument("--seed", type=int, default=42) - args = ap.parse_args() - - rows = collect(args.data) - if not rows: - raise SystemExit(f"No images found under {args.data}") - - splits = split(rows, args.seed) - args.out.parent.mkdir(parents=True, exist_ok=True) - with args.out.open("w", newline="", encoding="utf-8") as f: - w = csv.writer(f) - w.writerow(["path", "label", "source", "split"]) - for name, items in splits.items(): - for path, label, source in items: - w.writerow([path, label, source, name]) - - summary = {k: len(v) for k, v in splits.items()} - print(f"Manifest: {args.out}") - print(f"Totals: {summary} (overall {sum(summary.values())})") - - -if __name__ == "__main__": - main() diff --git a/datasets/download_dfdc_sample.py b/datasets/download_dfdc_sample.py deleted file mode 100644 index 290f639f16430ea03bec20cad71c34b4f4a3a898..0000000000000000000000000000000000000000 --- a/datasets/download_dfdc_sample.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Download a sample of the DFDC (Deepfake Detection Challenge) Preview dataset. - -The full DFDC is ~470GB; the *preview* release (~5GB, Kaggle) is enough for -diversity augmentation alongside FFPP. - -Requires the Kaggle CLI (`pip install kaggle`) and ~/.kaggle/kaggle.json. - -Usage: - python -m backend.training.datasets.download_dfdc_sample --output ./data/dfdc_preview -""" -from __future__ import annotations - -import argparse -import shutil -import subprocess -import sys -from pathlib import Path - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--output", required=True, type=Path) - ap.add_argument( - "--competition", - default="deepfake-detection-challenge", - help="Kaggle competition slug (default: deepfake-detection-challenge preview).", - ) - args = ap.parse_args() - - kaggle = shutil.which("kaggle") - if kaggle is None: - print("Kaggle CLI not found. Install with: pip install kaggle", file=sys.stderr) - print("Then place kaggle.json in ~/.kaggle/ (chmod 600).", file=sys.stderr) - sys.exit(2) - - args.output.mkdir(parents=True, exist_ok=True) - cmd = [kaggle, "competitions", "download", "-c", args.competition, "-p", str(args.output)] - print("Running:", " ".join(cmd)) - subprocess.run(cmd, check=True) - print(f"Downloaded to {args.output}. Unzip with: unzip *.zip") - - -if __name__ == "__main__": - main() diff --git a/datasets/download_ffhq.py b/datasets/download_ffhq.py deleted file mode 100644 index 9aad01da57b77488e4d3113295cd2769c3376826..0000000000000000000000000000000000000000 --- a/datasets/download_ffhq.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Download the FFHQ 128x128 thumbnail subset from the official Google Drive mirror. - -Pulls up to N images (default 10k) into the `real` bucket of the training set. -Falls back to the NVlabs 'ffhq-dataset' helper if available; otherwise expects -user to run the manual download once. - -Usage: - python -m backend.training.datasets.download_ffhq --output ./data/real/ffhq -n 10000 -""" -from __future__ import annotations - -import argparse -import shutil -import subprocess -import sys -from pathlib import Path - - -def try_nvlabs_helper(output: Path, num: int) -> bool: - """Prefer the official ffhq-dataset downloader if installed.""" - helper = shutil.which("ffhq-dataset") - if helper is None: - return False - cmd = [helper, "--json", "ffhq-dataset-v2.json", "--thumbs", "--num_threads", "4"] - print("Running:", " ".join(cmd)) - subprocess.run(cmd, cwd=output, check=False) - return True - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--output", required=True, type=Path) - ap.add_argument("-n", "--num", type=int, default=10000) - args = ap.parse_args() - args.output.mkdir(parents=True, exist_ok=True) - - if try_nvlabs_helper(args.output, args.num): - return - - print("[!] `ffhq-dataset` helper not installed.") - print(" Install via: pip install ffhq-dataset (requires gdown)") - print(" Or download thumbnails128x128.zip manually from:") - print(" https://github.com/NVlabs/ffhq-dataset") - print(f" Extract into: {args.output}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/datasets/extract_frames.py b/datasets/extract_frames.py deleted file mode 100644 index 28bebebb62c2676744a72e1f68ffdf4f5d74c2f4..0000000000000000000000000000000000000000 --- a/datasets/extract_frames.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Convert FFPP / DFDC videos -> 16 sampled frames at 224x224 RGB. - -Usage: - python -m backend.training.datasets.extract_frames \ - --input ./ffpp_data/original_sequences/youtube/raw/videos \ - --output ./ffpp_data/frames/real \ - --label real --frames 16 --size 224 -""" -from __future__ import annotations - -import argparse -import csv -from pathlib import Path - -import cv2 -import numpy as np -from tqdm import tqdm - - -def sample_frame_indices(total: int, n: int) -> list[int]: - if total <= 0: - return [] - if total <= n: - return list(range(total)) - step = total / float(n) - return [min(total - 1, int(step * i + step / 2)) for i in range(n)] - - -def extract_from_video(path: Path, out_dir: Path, n: int, size: int) -> int: - cap = cv2.VideoCapture(str(path)) - if not cap.isOpened(): - return 0 - total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - indices = set(sample_frame_indices(total, n)) - out_dir.mkdir(parents=True, exist_ok=True) - - saved = 0 - i = 0 - while True: - ok, frame = cap.read() - if not ok: - break - if i in indices: - frame = cv2.resize(frame, (size, size), interpolation=cv2.INTER_AREA) - cv2.imwrite(str(out_dir / f"{path.stem}_f{i:06d}.jpg"), frame, [cv2.IMWRITE_JPEG_QUALITY, 95]) - saved += 1 - i += 1 - cap.release() - return saved - - -def main() -> None: - ap = argparse.ArgumentParser(description="Sample N frames per video and resize.") - ap.add_argument("--input", required=True, type=Path, help="Directory of .mp4 videos (recursive).") - ap.add_argument("--output", required=True, type=Path, help="Directory to write .jpg frames.") - ap.add_argument("--label", required=True, choices=["real", "fake"], help="Label tag for manifest.") - ap.add_argument("--frames", type=int, default=16) - ap.add_argument("--size", type=int, default=224) - ap.add_argument("--manifest", type=Path, default=None, help="Optional CSV manifest append path.") - args = ap.parse_args() - - videos = [p for p in args.input.rglob("*.mp4")] - if not videos: - print(f"No .mp4 found under {args.input}") - return - - rows: list[tuple[str, str, str]] = [] - total_frames = 0 - for vid in tqdm(videos, desc=f"extract[{args.label}]"): - rel_out = args.output / vid.stem - saved = extract_from_video(vid, rel_out, args.frames, args.size) - total_frames += saved - if args.manifest is not None: - for jpg in rel_out.glob("*.jpg"): - rows.append((str(jpg), args.label, vid.stem)) - - if args.manifest is not None and rows: - args.manifest.parent.mkdir(parents=True, exist_ok=True) - new_file = not args.manifest.exists() - with args.manifest.open("a", newline="", encoding="utf-8") as f: - w = csv.writer(f) - if new_file: - w.writerow(["path", "label", "source_video"]) - w.writerows(rows) - - print(f"Done. Videos: {len(videos)}, frames written: {total_frames}") - - -if __name__ == "__main__": - main() diff --git a/datasets/procure_all.ps1 b/datasets/procure_all.ps1 deleted file mode 100644 index 4edafaffd1c99062de0244d2c7a3db1cf6813cc6..0000000000000000000000000000000000000000 --- a/datasets/procure_all.ps1 +++ /dev/null @@ -1,40 +0,0 @@ -# Phase 11.1 orchestrator for Windows (PowerShell) -$ErrorActionPreference = "Stop" - -$ROOT = if ($env:ROOT) { $env:ROOT } else { ".\data" } -$FFPP = if ($env:FFPP) { $env:FFPP } else { ".\ffpp_data" } - -New-Item -ItemType Directory -Force -Path "$ROOT\real" | Out-Null -New-Item -ItemType Directory -Force -Path "$ROOT\fake" | Out-Null -New-Item -ItemType Directory -Force -Path $FFPP | Out-Null - -Write-Host "1. FaceForensics++ (highly compressed c40, 10 videos only) -- requires TOS keypress" -python backend\scripts\download_ffpp.py $FFPP -d all -c c40 -t videos -n 10 - -Write-Host "2. Frame extraction: real (original youtube)" -python -m backend.training.datasets.extract_frames ` - --input "$FFPP\original_sequences\youtube\c40\videos" ` - --output "$ROOT\real\ffpp_youtube" --label real --frames 4 --size 224 - -Write-Host "3. Frame extraction: fakes (each manipulation family)" -$Families = @("Deepfakes", "Face2Face", "FaceSwap", "NeuralTextures", "FaceShifter") -foreach ($fam in $Families) { - $famLower = $fam.ToLower() - python -m backend.training.datasets.extract_frames ` - --input "$FFPP\manipulated_sequences\$fam\c40\videos" ` - --output "$ROOT\fake\ffpp_$famLower" --label fake --frames 4 --size 224 -} - -Write-Host "4. FFHQ thumbnails (real - limited to 100 items)" -python -m backend.training.datasets.download_ffhq --output "$ROOT\real\ffhq" -n 100 - - -Write-Host "6. DFDC preview sample (fake+real)" -python -m backend.training.datasets.download_dfdc_sample --output "$ROOT\_dfdc_raw" -Write-Host "NOTE: You will need to manually unzip + sort DFDC into $ROOT\fake\dfdc and $ROOT\real\dfdc" - -Write-Host "7. Build manifest" -python -m backend.training.datasets.build_manifest ` - --data $ROOT --out "$ROOT\manifest.csv" --seed 42 - -Write-Host "Phase 11.1 complete. See $ROOT\manifest.csv" diff --git a/datasets/procure_all.sh b/datasets/procure_all.sh deleted file mode 100644 index fa2f94ecf2959a92df56d31572c4c26c9256b0d1..0000000000000000000000000000000000000000 --- a/datasets/procure_all.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# Phase 11.1 orchestrator: download + frame-extract + manifest. -# Total disk target: ~120k labeled images. Expect 60-80GB intermediate, ~30GB frames. - -set -euo pipefail - -ROOT="${ROOT:-./data}" -FFPP="${FFPP:-./ffpp_data}" -mkdir -p "$ROOT/real" "$ROOT/fake" "$FFPP" - -# 1. FaceForensics++ (raw, videos) -- requires TOS keypress -python backend/scripts/download_ffpp.py "$FFPP" -d all -c raw -t videos - -# 2. Frame extraction: real (original youtube) -python -m backend.training.datasets.extract_frames \ - --input "$FFPP/original_sequences/youtube/raw/videos" \ - --output "$ROOT/real/ffpp_youtube" --label real --frames 16 --size 224 - -# 3. Frame extraction: fakes (each manipulation family) -for fam in Deepfakes Face2Face FaceSwap NeuralTextures FaceShifter; do - python -m backend.training.datasets.extract_frames \ - --input "$FFPP/manipulated_sequences/$fam/raw/videos" \ - --output "$ROOT/fake/ffpp_${fam,,}" --label fake --frames 16 --size 224 -done - -# 4. FFHQ thumbnails (real) -python -m backend.training.datasets.download_ffhq --output "$ROOT/real/ffhq" -n 10000 - -# 6. DFDC preview sample (fake+real) -- needs Kaggle creds -python -m backend.training.datasets.download_dfdc_sample --output "$ROOT/_dfdc_raw" -# NOTE: unzip + sort into $ROOT/fake/dfdc and $ROOT/real/dfdc per DFDC metadata.json - -# 7. Build manifest -python -m backend.training.datasets.build_manifest \ - --data "$ROOT" --out "$ROOT/manifest.csv" --seed 42 - -echo "Phase 11.1 complete. See $ROOT/manifest.csv" diff --git a/db/database.py b/db/database.py index 8046cb2cec21124d47ad3a816598c84b5f34f19e..79748240b5de4b5431e34553369555c7c8b18047 100644 --- a/db/database.py +++ b/db/database.py @@ -1,28 +1,26 @@ -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.orm import DeclarativeBase, sessionmaker from config import settings -_is_postgres = not settings.DATABASE_URL.startswith("sqlite") - engine = create_engine( settings.DATABASE_URL, - # SQLite needs check_same_thread=False; Postgres doesn't support it - connect_args={"check_same_thread": False} if not _is_postgres else {}, - # Neon (and other serverless Postgres) silently drops idle SSL connections. - # pool_pre_ping=True: test each connection before use and transparently - # reconnect if the server closed it — eliminates "SSL connection has been - # closed unexpectedly" 500s. - pool_pre_ping=_is_postgres, - # Recycle connections every 5 min so we never hold a connection past Neon's - # idle timeout (~5–10 min depending on plan). - pool_recycle=300 if _is_postgres else -1, - # Keep pool small — HF free tier is single-process; Neon free tier has a - # max-connection limit. - pool_size=5 if _is_postgres else 5, - max_overflow=2 if _is_postgres else 10, + connect_args={"check_same_thread": False} if settings.DATABASE_URL.startswith("sqlite") else {}, + pool_pre_ping=True, + pool_recycle=300, ) + +if settings.DATABASE_URL.startswith("sqlite"): + @event.listens_for(engine, "connect") + def _sqlite_on_connect(dbapi_conn, _): + # Enforce FK constraints (needed for ON DELETE SET NULL) + WAL for better + # concurrent reads while a writer is active. + cur = dbapi_conn.cursor() + cur.execute("PRAGMA foreign_keys=ON") + cur.execute("PRAGMA journal_mode=WAL") + cur.close() + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -40,5 +38,31 @@ def get_db(): def init_db(): from db import models # noqa: F401 + from sqlalchemy import inspect, text + Base.metadata.create_all(bind=engine) + # Phase 19.4 — lightweight in-place migration for new columns. + # Alembic is overkill here; just ALTER TABLE when a new column is missing. + insp = inspect(engine) + if "analyses" in insp.get_table_names(): + existing = {c["name"] for c in insp.get_columns("analyses")} + additions = { + "media_hash": "VARCHAR(64)", + "media_path": "VARCHAR(512)", + "thumbnail_url": "VARCHAR(512)", + } + with engine.begin() as conn: + for col, ddl in additions.items(): + if col not in existing: + conn.execute(text(f"ALTER TABLE analyses ADD COLUMN {col} {ddl}")) + # Indices (CREATE INDEX IF NOT EXISTS is SQLite+Postgres safe) + for ddl in ( + "CREATE INDEX IF NOT EXISTS ix_analyses_media_hash ON analyses (media_hash)", + "CREATE INDEX IF NOT EXISTS ix_record_user_created ON analyses (user_id, created_at)", + "CREATE INDEX IF NOT EXISTS ix_report_analysis ON reports (analysis_id)", + ): + try: + conn.execute(text(ddl)) + except Exception: # noqa: BLE001 + pass diff --git a/db/models.py b/db/models.py index af3b2f8f14b6485f08ed933ec490c11d10802e4a..d4a88292cc7096f802685aa61a79014cfddd634e 100644 --- a/db/models.py +++ b/db/models.py @@ -1,6 +1,6 @@ -from datetime import datetime +from datetime import datetime, timezone -from sqlalchemy import DateTime, ForeignKey, Integer, String, Text +from sqlalchemy import DateTime, ForeignKey, Index, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column, relationship from db.database import Base @@ -13,7 +13,7 @@ class User(Base): email: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False) password_hash: Mapped[str] = mapped_column(String(255), nullable=False) name: Mapped[str | None] = mapped_column(String(255), nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) analyses: Mapped[list["AnalysisRecord"]] = relationship(back_populates="user") @@ -22,16 +22,26 @@ class AnalysisRecord(Base): __tablename__ = "analyses" id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) - user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), nullable=True, + ) media_type: Mapped[str] = mapped_column(String(32), nullable=False) # image|video|text|screenshot verdict: Mapped[str] = mapped_column(String(32), nullable=False) authenticity_score: Mapped[float] = mapped_column(nullable=False) result_json: Mapped[str] = mapped_column(Text, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + # Phase 19.1 / 19.2 — SHA-256 dedup + object storage + media_hash: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) + media_path: Mapped[str | None] = mapped_column(String(512), nullable=True) + thumbnail_url: Mapped[str | None] = mapped_column(String(512), nullable=True) user: Mapped["User | None"] = relationship(back_populates="analyses") report: Mapped["Report | None"] = relationship(back_populates="analysis", uselist=False) + __table_args__ = ( + Index("ix_record_user_created", "user_id", "created_at"), + ) + class Report(Base): __tablename__ = "reports" @@ -39,7 +49,11 @@ class Report(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) analysis_id: Mapped[int] = mapped_column(ForeignKey("analyses.id"), nullable=False) file_path: Mapped[str] = mapped_column(String(512), nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) - expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) analysis: Mapped["AnalysisRecord"] = relationship(back_populates="report") + + __table_args__ = ( + Index("ix_report_analysis", "analysis_id"), + ) diff --git a/deepshield.db-shm b/deepshield.db-shm new file mode 100644 index 0000000000000000000000000000000000000000..134136c1f326bffa9f6ca9cf6afa24f4b679ace3 Binary files /dev/null and b/deepshield.db-shm differ diff --git a/deepshield.db-wal b/deepshield.db-wal new file mode 100644 index 0000000000000000000000000000000000000000..825a1b9b9240aa1a081ce3635db6f287bd20a80a Binary files /dev/null and b/deepshield.db-wal differ diff --git a/deepshield_13_5bcf1328.pdf b/deepshield_13_5bcf1328.pdf deleted file mode 100644 index dee4c670b4e4e2064488fe79f0eec4ac48bf39ea..0000000000000000000000000000000000000000 --- a/deepshield_13_5bcf1328.pdf +++ /dev/null @@ -1,148 +0,0 @@ -%PDF-1.4 -% ReportLab Generated PDF document (opensource) -1 0 obj -<< -/F1 2 0 R /F2 3 0 R /F3 5 0 R ->> -endobj -2 0 obj -<< -/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font ->> -endobj -3 0 obj -<< -/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font ->> -endobj -4 0 obj -<< -/Contents 18 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 17 0 R /Resources << -/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] ->> /Rotate 0 /Trans << - ->> - /Type /Page ->> -endobj -5 0 obj -<< -/BaseFont /Symbol /Name /F3 /Subtype /Type1 /Type /Font ->> -endobj -6 0 obj -<< -/Contents 19 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 17 0 R /Resources << -/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] ->> /Rotate 0 /Trans << - ->> - /Type /Page ->> -endobj -7 0 obj -<< -/Outlines 9 0 R /PageMode /UseNone /Pages 17 0 R /Type /Catalog ->> -endobj -8 0 obj -<< -/Author () /CreationDate (D:20260415181653+05'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260415181653+05'00') /Producer (xhtml2pdf ) - /Subject () /Title (DeepShield Analysis Report \204 7771f496-45b1-4c97-8a1a-d9d2492ca67d) /Trapped /False ->> -endobj -9 0 obj -<< -/Count 3 /First 10 0 R /Last 10 0 R /Type /Outlines ->> -endobj -10 0 obj -<< -/Count -4 /Dest [ 4 0 R /Fit ] /First 11 0 R /Last 16 0 R /Parent 9 0 R /Title (DeepShield Analysis Report) ->> -endobj -11 0 obj -<< -/Dest [ 4 0 R /Fit ] /Next 12 0 R /Parent 10 0 R /Title (Verdict) ->> -endobj -12 0 obj -<< -/Count -2 /Dest [ 4 0 R /Fit ] /First 13 0 R /Last 14 0 R /Next 15 0 R /Parent 10 0 R - /Prev 11 0 R /Title (Text Classification) ->> -endobj -13 0 obj -<< -/Dest [ 4 0 R /Fit ] /Next 14 0 R /Parent 12 0 R /Title (Sensationalism Signals) ->> -endobj -14 0 obj -<< -/Dest [ 4 0 R /Fit ] /Parent 12 0 R /Prev 13 0 R /Title (Extracted Keywords) ->> -endobj -15 0 obj -<< -/Dest [ 4 0 R /Fit ] /Next 16 0 R /Parent 10 0 R /Prev 12 0 R /Title (Trusted Source Cross-Reference \(1\)) ->> -endobj -16 0 obj -<< -/Dest [ 6 0 R /Fit ] /Parent 10 0 R /Prev 15 0 R /Title (Processing Summary) ->> -endobj -17 0 obj -<< -/Count 2 /Kids [ 4 0 R 6 0 R ] /Type /Pages ->> -endobj -18 0 obj -<< -/Filter [ /ASCII85Decode /FlateDecode ] /Length 1750 ->> -stream -Gb"/(9lo&I&A@sBlm4G[Acr2Y4p^$ca2t\gAsuiHo\c,I9gURE8lSA3M>qu?,XkR;()9nE&%0G$"Ts\%gUFdJ0E[3iXSb#I!k]Slq-+&^_fu5V&-:f'>`[5155TjpXI_!]U"iQd1qrcX0jNK021sk.K_S`f[kfkaR[pr2$LLU)UX&`3>7R17rJ3t':B_<4Kk*Grr8\a:5/Z<<[I]mbfHq28c@Y+3O)t)0k@mu0K^fiq^N*(u.%T.'jlS?L^o+>>SgBV8H:sX>5A0-l`)&\h4Lk6L5I=)ArV#_bh%^>M_c,"jSErfH[2A&CfKtLn_&K3h)!u;:i'6.H*(apE@/QWkIgF*OaTZ"ZT=me'_?iN-hL[(uHeb"'/B!\/7d068ieW>Y3P8NcsU#;"%eOe_!^-"Xsc?9a'H,u4"nMEm$3F[>c1S8J!`Sh;Ye8pG>de>ac3KpI*&j-(`*[@OB&i#OgJSl=(I-';4Vs.^rc%L+kt99^Gd]mfUsWoLD02jLH*WUl.Pb(oF^j?7RUN!m&Us22M!@Ald**8+J._-f-FEVm$t<`HO6GNqd_[bhJ&8qK0d-ZKt;EB60ud0-2Z:*Z]IT(dG)'7QU\#u^ecY/FgdnO#RWf_=Js*t;iiO?'fQ:g&@nC/Xhu.;&o1b+?_6-Z%i4;1H5GAUag0*4LfL'2;Sl`["O/H6p>jU\SO4%Ffq^-']muUp/PKbuj>J71&Mh5t,WF_k&]O@P+do^;.WV"r6Kkb#5`,aF$-adPdc+'072](pse[q;.^?I#Q#kci1Qr9Z_U:Q_lQ53n!nIBHrchNfMeP-HF*=<22XdSrZ8j>sP4CR1SEP\Ge.aCh(VEW.)F'<]`"gVnaq<<]K,.uCIMlUqSgV3UTe;V8("S^2/7e`3>4E]],alEY#@T-dG.(=/^7(s[bh3%omN/'WKl<"q_K`T7$VrMt.GfckX6]1EfAB]1F6o6g>\:2Etf)rD.XNrRc2pgl"Hr<(1MCd%~>endstream -endobj -19 0 obj -<< -/Filter [ /ASCII85Decode /FlateDecode ] /Length 1251 ->> -stream -Gau`R;01GN&:Vs/fU'm&SZsB\Z>@pd[^l$Ne'"!6Hco+&(^1n.H%;Q95P;kU[/"Vgs.N%@'=M6kAJN1afF&?E_+rA+1KE+S:4],1QpOr^qg01e<#d,;@\e=!\1-*,1T[41J&^DSg86dC5.#&+tMiZhie$%p]f=sWJ!9ni#^ZR?Gp5lVJY,MW+]dGA*V*5[2WS\gs>9t%t32b/^W)[_+r7&3kOLD>8WTI508QU_ZkVRb*l"j_,ie@Wk/$,J'=rjAsRr^aIAp,g4N\@rcW@_7fV)G7.f:C\2aDCnK2"(-Yh-fNKV4ogPJ_Bbno/AG^W)=l`02mHESBSd,2MW2Q,8S^O,7f_^Pj+'$c\[n!'TZ'8A[[6$M/6Vlo9egXU318J0Zl;rXSYgM=-\-3TecfRc]m]FKNI.=E4amT3\PSaWQi;TtrPVN"#t`E;bkNM&M.:/OC)MK2$$?Jp$`SY/%t"jbj6*+.%6.71qjEsp)j@\0#RIF/1!&^q"O7Ou;8DL^2(?$>18.AWa`H$Fi,Ak&SQPl+Y^;rG>nArp/_q%9B[r]_;\_^p'[__7OH7)iuf]c[rld?RB/MrP3T8Xk7VY%=qG1""FA,mioCp,lF3^-AZtKRg/NFX>&kA^rZpnFAendstream -endobj -xref -0 20 -0000000000 65535 f -0000000061 00000 n -0000000112 00000 n -0000000219 00000 n -0000000331 00000 n -0000000536 00000 n -0000000613 00000 n -0000000818 00000 n -0000000903 00000 n -0000001223 00000 n -0000001296 00000 n -0000001426 00000 n -0000001514 00000 n -0000001667 00000 n -0000001770 00000 n -0000001869 00000 n -0000001999 00000 n -0000002098 00000 n -0000002164 00000 n -0000004006 00000 n -trailer -<< -/ID -[<8e273c2672d813e3cd44109eb1edd604><8e273c2672d813e3cd44109eb1edd604>] -% ReportLab generated PDF document -- digest (opensource) - -/Info 8 0 R -/Root 7 0 R -/Size 20 ->> -startxref -5349 -%%EOF diff --git a/deps.py b/deps.py deleted file mode 100644 index 776c7ceb8b1184c3e0e03627b7757a6b34e2bc7b..0000000000000000000000000000000000000000 --- a/deps.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -from fastapi import Depends, Header, HTTPException, status -from sqlalchemy.orm import Session - -from db.database import get_db -from db.models import User -from services.auth_service import decode_token, get_user - - -def _extract_bearer(authorization: str | None) -> str | None: - if not authorization: - return None - parts = authorization.split() - if len(parts) != 2 or parts[0].lower() != "bearer": - return None - return parts[1] - - -def get_current_user( - authorization: str | None = Header(default=None), - db: Session = Depends(get_db), -) -> User: - token = _extract_bearer(authorization) - if not token: - raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Missing bearer token") - payload = decode_token(token) - if not payload or "sub" not in payload: - raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid or expired token") - user = get_user(db, int(payload["sub"])) - if not user: - raise HTTPException(status.HTTP_401_UNAUTHORIZED, "User not found") - return user - - -def optional_current_user( - authorization: str | None = Header(default=None), - db: Session = Depends(get_db), -) -> User | None: - token = _extract_bearer(authorization) - if not token: - return None - payload = decode_token(token) - if not payload or "sub" not in payload: - return None - return get_user(db, int(payload["sub"])) diff --git a/download_ffpp.py b/download_ffpp.py deleted file mode 100644 index 4b86da46b8afaa685f312a5e6617e8dd75add856..0000000000000000000000000000000000000000 --- a/download_ffpp.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python -""" Downloads FaceForensics++ and Deep Fake Detection public data release -Example usage: - see -h or https://github.com/ondyari/FaceForensics -""" -# -*- coding: utf-8 -*- -import argparse -import os -import urllib -import urllib.request -import tempfile -import time -import sys -import json -import random -from tqdm import tqdm -from os.path import join - - -# URLs and filenames -FILELIST_URL = 'misc/filelist.json' -DEEPFEAKES_DETECTION_URL = 'misc/deepfake_detection_filenames.json' -DEEPFAKES_MODEL_NAMES = ['decoder_A.h5', 'decoder_B.h5', 'encoder.h5',] - -# Parameters -DATASETS = { - 'original_youtube_videos': 'misc/downloaded_youtube_videos.zip', - 'original_youtube_videos_info': 'misc/downloaded_youtube_videos_info.zip', - 'original': 'original_sequences/youtube', - 'DeepFakeDetection_original': 'original_sequences/actors', - 'Deepfakes': 'manipulated_sequences/Deepfakes', - 'DeepFakeDetection': 'manipulated_sequences/DeepFakeDetection', - 'Face2Face': 'manipulated_sequences/Face2Face', - 'FaceShifter': 'manipulated_sequences/FaceShifter', - 'FaceSwap': 'manipulated_sequences/FaceSwap', - 'NeuralTextures': 'manipulated_sequences/NeuralTextures' - } -ALL_DATASETS = ['original', 'DeepFakeDetection_original', 'Deepfakes', - 'DeepFakeDetection', 'Face2Face', 'FaceShifter', 'FaceSwap', - 'NeuralTextures'] -COMPRESSION = ['raw', 'c23', 'c40'] -TYPE = ['videos', 'masks', 'models'] -SERVERS = ['EU', 'EU2', 'CA'] - - -def parse_args(): - parser = argparse.ArgumentParser( - description='Downloads FaceForensics v2 public data release.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument('output_path', type=str, help='Output directory.') - parser.add_argument('-d', '--dataset', type=str, default='all', - help='Which dataset to download, either pristine or ' - 'manipulated data or the downloaded youtube ' - 'videos.', - choices=list(DATASETS.keys()) + ['all'] - ) - parser.add_argument('-c', '--compression', type=str, default='raw', - help='Which compression degree. All videos ' - 'have been generated with h264 with a varying ' - 'codec. Raw (c0) videos are lossless compressed.', - choices=COMPRESSION - ) - parser.add_argument('-t', '--type', type=str, default='videos', - help='Which file type, i.e. videos, masks, for our ' - 'manipulation methods, models, for Deepfakes.', - choices=TYPE - ) - parser.add_argument('-n', '--num_videos', type=int, default=None, - help='Select a number of videos number to ' - "download if you don't want to download the full" - ' dataset.') - parser.add_argument('--server', type=str, default='EU', - help='Server to download the data from. If you ' - 'encounter a slow download speed, consider ' - 'changing the server.', - choices=SERVERS - ) - args = parser.parse_args() - - # URLs - server = args.server - if server == 'EU': - server_url = 'http://canis.vc.in.tum.de:8100/' - elif server == 'EU2': - server_url = 'http://kaldir.vc.in.tum.de/faceforensics/' - elif server == 'CA': - server_url = 'http://falas.cmpt.sfu.ca:8100/' - else: - raise Exception('Wrong server name. Choices: {}'.format(str(SERVERS))) - args.tos_url = server_url + 'webpage/FaceForensics_TOS.pdf' - args.base_url = server_url + 'v3/' - args.deepfakes_model_url = server_url + 'v3/manipulated_sequences/' + \ - 'Deepfakes/models/' - - return args - - -def download_files(filenames, base_url, output_path, report_progress=True): - os.makedirs(output_path, exist_ok=True) - if report_progress: - filenames = tqdm(filenames) - for filename in filenames: - download_file(base_url + filename, join(output_path, filename)) - - -def reporthook(count, block_size, total_size): - global start_time - if count == 0: - start_time = time.time() - return - duration = time.time() - start_time - progress_size = int(count * block_size) - speed = int(progress_size / (1024 * duration)) - percent = int(count * block_size * 100 / total_size) - sys.stdout.write("\rProgress: %d%%, %d MB, %d KB/s, %d seconds passed" % - (percent, progress_size / (1024 * 1024), speed, duration)) - sys.stdout.flush() - - -def download_file(url, out_file, report_progress=False): - out_dir = os.path.dirname(out_file) - if not os.path.isfile(out_file): - fh, out_file_tmp = tempfile.mkstemp(dir=out_dir) - f = os.fdopen(fh, 'w') - f.close() - if report_progress: - urllib.request.urlretrieve(url, out_file_tmp, - reporthook=reporthook) - else: - urllib.request.urlretrieve(url, out_file_tmp) - os.rename(out_file_tmp, out_file) - else: - tqdm.write('WARNING: skipping download of existing file ' + out_file) - - -def main(args): - # TOS - print('By pressing any key to continue you confirm that you have agreed '\ - 'to the FaceForensics terms of use as described at:') - print(args.tos_url) - print('***') - print('Press any key to continue, or CTRL-C to exit.') - _ = input('') - - # Extract arguments - c_datasets = [args.dataset] if args.dataset != 'all' else ALL_DATASETS - c_type = args.type - c_compression = args.compression - num_videos = args.num_videos - output_path = args.output_path - os.makedirs(output_path, exist_ok=True) - - # Check for special dataset cases - for dataset in c_datasets: - dataset_path = DATASETS[dataset] - # Special cases - if 'original_youtube_videos' in dataset: - # Here we download the original youtube videos zip file - print('Downloading original youtube videos.') - if not 'info' in dataset_path: - print('Please be patient, this may take a while (~40gb)') - suffix = '' - else: - suffix = 'info' - download_file(args.base_url + '/' + dataset_path, - out_file=join(output_path, - 'downloaded_videos{}.zip'.format( - suffix)), - report_progress=True) - return - - # Else: regular datasets - print('Downloading {} of dataset "{}"'.format( - c_type, dataset_path - )) - - # Get filelists and video lenghts list from server - if 'DeepFakeDetection' in dataset_path or 'actors' in dataset_path: - filepaths = json.loads(urllib.request.urlopen(args.base_url + '/' + - DEEPFEAKES_DETECTION_URL).read().decode("utf-8")) - if 'actors' in dataset_path: - filelist = filepaths['actors'] - else: - filelist = filepaths['DeepFakesDetection'] - elif 'original' in dataset_path: - # Load filelist from server - file_pairs = json.loads(urllib.request.urlopen(args.base_url + '/' + - FILELIST_URL).read().decode("utf-8")) - filelist = [] - for pair in file_pairs: - filelist += pair - else: - # Load filelist from server - file_pairs = json.loads(urllib.request.urlopen(args.base_url + '/' + - FILELIST_URL).read().decode("utf-8")) - # Get filelist - filelist = [] - for pair in file_pairs: - filelist.append('_'.join(pair)) - if c_type != 'models': - filelist.append('_'.join(pair[::-1])) - # Maybe limit number of videos for download - if num_videos is not None and num_videos > 0: - print('Downloading the first {} videos'.format(num_videos)) - filelist = filelist[:num_videos] - - # Server and local paths - dataset_videos_url = args.base_url + '{}/{}/{}/'.format( - dataset_path, c_compression, c_type) - dataset_mask_url = args.base_url + '{}/{}/videos/'.format( - dataset_path, 'masks', c_type) - - if c_type == 'videos': - dataset_output_path = join(output_path, dataset_path, c_compression, - c_type) - print('Output path: {}'.format(dataset_output_path)) - filelist = [filename + '.mp4' for filename in filelist] - download_files(filelist, dataset_videos_url, dataset_output_path) - elif c_type == 'masks': - dataset_output_path = join(output_path, dataset_path, c_type, - 'videos') - print('Output path: {}'.format(dataset_output_path)) - if 'original' in dataset: - if args.dataset != 'all': - print('Only videos available for original data. Aborting.') - return - else: - print('Only videos available for original data. ' - 'Skipping original.\n') - continue - if 'FaceShifter' in dataset: - print('Masks not available for FaceShifter. Aborting.') - return - filelist = [filename + '.mp4' for filename in filelist] - download_files(filelist, dataset_mask_url, dataset_output_path) - - # Else: models for deepfakes - else: - if dataset != 'Deepfakes' and c_type == 'models': - print('Models only available for Deepfakes. Aborting') - return - dataset_output_path = join(output_path, dataset_path, c_type) - print('Output path: {}'.format(dataset_output_path)) - - # Get Deepfakes models - for folder in tqdm(filelist): - folder_filelist = DEEPFAKES_MODEL_NAMES - - # Folder paths - folder_base_url = args.deepfakes_model_url + folder + '/' - folder_dataset_output_path = join(dataset_output_path, - folder) - download_files(folder_filelist, folder_base_url, - folder_dataset_output_path, - report_progress=False) # already done - - -if __name__ == "__main__": - args = parse_args() - main(args) diff --git a/ela_service.py b/ela_service.py deleted file mode 100644 index e937502d11a21c347c224611a155047d8c88bbfc..0000000000000000000000000000000000000000 --- a/ela_service.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Error Level Analysis (ELA) — Phase 12.1 - -Re-saves an image at a fixed JPEG quality and diffs against the original to reveal -per-pixel manipulation artifacts. Regions that were recently edited will show -higher error levels than untouched areas. -""" - -from __future__ import annotations - -import base64 -import io - -import cv2 -import numpy as np -from loguru import logger -from PIL import Image - - -def _compute_ela(pil_img: Image.Image, quality: int = 90, scale: float = 15.0) -> np.ndarray: - """Return an ELA difference map as a uint8 (H,W,3) RGB array. - - Args: - pil_img: Input image (any format — converted to RGB internally). - quality: JPEG re-save quality level (lower = more aggressive compression). - scale: Amplification factor for the difference (higher = more contrast). - - Returns: - Difference image as uint8 (H,W,3) array. - """ - rgb = pil_img.convert("RGB") - - # Re-save at specified JPEG quality into an in-memory buffer - buf = io.BytesIO() - rgb.save(buf, format="JPEG", quality=quality) - buf.seek(0) - resaved = Image.open(buf).convert("RGB") - - original_arr = np.array(rgb, dtype=np.float32) - resaved_arr = np.array(resaved, dtype=np.float32) - - # Per-pixel absolute difference, amplified - diff = np.abs(original_arr - resaved_arr) * scale - diff = np.clip(diff, 0, 255).astype(np.uint8) - - return diff - - -def generate_ela_base64(pil_img: Image.Image, quality: int = 90, scale: float = 15.0) -> str: - """Produce a base64 data-URL PNG of the ELA difference map. - - Regions with higher error levels (brighter in the output) are more likely - to have been digitally manipulated. - """ - diff = _compute_ela(pil_img, quality=quality, scale=scale) - - buf = io.BytesIO() - Image.fromarray(diff).save(buf, format="PNG") - b64 = base64.b64encode(buf.getvalue()).decode("ascii") - - logger.info(f"ELA map generated ({diff.shape[1]}x{diff.shape[0]})") - return f"data:image/png;base64,{b64}" - - -def generate_blended_ela_base64( - pil_img: Image.Image, - gradcam_weight: float = 0.6, - ela_weight: float = 0.4, - quality: int = 90, - scale: float = 15.0, -) -> str: - """Blend Grad-CAM heatmap overlay with ELA at specified weights. - - This is a utility for the 'blended' mode — it composites the ELA - difference map on top of the original image for visual clarity. - """ - rgb = pil_img.convert("RGB") - original_arr = np.array(rgb, dtype=np.float32) - ela_arr = _compute_ela(pil_img, quality=quality, scale=scale).astype(np.float32) - - # Blend: overlay ELA on the original for visual context - blended = np.clip(original_arr * 0.5 + ela_arr * 0.5, 0, 255).astype(np.uint8) - - buf = io.BytesIO() - Image.fromarray(blended).save(buf, format="PNG") - b64 = base64.b64encode(buf.getvalue()).decode("ascii") - - logger.info(f"Blended ELA generated ({blended.shape[1]}x{blended.shape[0]})") - return f"data:image/png;base64,{b64}" diff --git a/exif_service.py b/exif_service.py deleted file mode 100644 index 61c69ef39ff455182b5b1f0ab151f0d5aac8bbff..0000000000000000000000000000000000000000 --- a/exif_service.py +++ /dev/null @@ -1,129 +0,0 @@ -"""EXIF Metadata Extraction — Phase 12.2 - -Extracts camera metadata from uploaded images and computes a trust adjustment -score: presence of authentic camera metadata lowers fake probability, while -evidence of editing software raises it. -""" - -from __future__ import annotations - -from typing import Optional - -from loguru import logger -from PIL import Image -from PIL.ExifTags import TAGS, GPSTAGS - -from schemas.common import ExifSummary - - -# Software strings that suggest post-processing / generation -_SUSPICIOUS_SOFTWARE = { - "adobe photoshop", "photoshop", "gimp", "affinity photo", - "stable diffusion", "midjourney", "dall-e", "comfyui", - "automatic1111", "invokeai", -} - -# Software strings that are normal camera firmware -_CAMERA_SOFTWARE = { - "ver.", "firmware", "camera", "dji", "gopro", -} - - -def _decode_gps(gps_info: dict) -> Optional[str]: - """Decode EXIF GPSInfo dict into a human-readable lat/lon string.""" - try: - def _to_decimal(values, ref): - d, m, s = [float(v) for v in values] - decimal = d + m / 60.0 + s / 3600.0 - if ref in ("S", "W"): - decimal = -decimal - return decimal - - lat = _to_decimal(gps_info.get(2, (0, 0, 0)), gps_info.get(1, "N")) - lon = _to_decimal(gps_info.get(4, (0, 0, 0)), gps_info.get(3, "E")) - return f"{lat:.6f}, {lon:.6f}" - except Exception: - return None - - -def extract_exif(pil_img: Image.Image, raw_bytes: bytes) -> ExifSummary: - """Extract EXIF metadata and compute a trust adjustment score. - - Trust adjustment logic: - - Valid Make + Model + DateTimeOriginal → -15 (more likely real camera photo) - - GPS info present → -5 additional (real photos often have GPS) - - Suspicious editing software detected → +10 (more likely manipulated) - - No EXIF at all → 0 (inconclusive — many platforms strip EXIF) - """ - summary = ExifSummary() - - try: - exif_data = pil_img._getexif() - except Exception: - exif_data = None - - if not exif_data: - # Try exifread as fallback for formats Pillow doesn't handle well - try: - import exifread - from io import BytesIO - tags = exifread.process_file(BytesIO(raw_bytes), details=False) - if tags: - summary.make = str(tags.get("Image Make", "")).strip() or None - summary.model = str(tags.get("Image Model", "")).strip() or None - summary.datetime_original = str(tags.get("EXIF DateTimeOriginal", "")).strip() or None - summary.software = str(tags.get("Image Software", "")).strip() or None - summary.lens_model = str(tags.get("EXIF LensModel", "")).strip() or None - except ImportError: - logger.debug("exifread not installed, skipping fallback EXIF extraction") - except Exception as e: - logger.debug(f"exifread fallback failed: {e}") - else: - # Decode Pillow EXIF - decoded = {} - for tag_id, value in exif_data.items(): - tag_name = TAGS.get(tag_id, tag_id) - decoded[tag_name] = value - - summary.make = str(decoded.get("Make", "")).strip() or None - summary.model = str(decoded.get("Model", "")).strip() or None - summary.datetime_original = str(decoded.get("DateTimeOriginal", "")).strip() or None - summary.software = str(decoded.get("Software", "")).strip() or None - summary.lens_model = str(decoded.get("LensModel", "")).strip() or None - - # GPS - gps_raw = decoded.get("GPSInfo") - if gps_raw and isinstance(gps_raw, dict): - gps_decoded = {} - for k, v in gps_raw.items(): - gps_decoded[GPSTAGS.get(k, k)] = v - summary.gps_info = _decode_gps(gps_decoded) - - # ── Trust adjustment scoring ── - adjustment = 0 - reasons = [] - - has_camera_meta = summary.make and summary.model and summary.datetime_original - if has_camera_meta: - adjustment -= 15 - reasons.append("valid camera metadata (Make/Model/DateTime)") - - if summary.gps_info: - adjustment -= 5 - reasons.append("GPS coordinates present") - - if summary.software: - sw_lower = summary.software.lower() - if any(s in sw_lower for s in _SUSPICIOUS_SOFTWARE): - adjustment += 10 - reasons.append(f"editing software detected: {summary.software}") - elif any(s in sw_lower for s in _CAMERA_SOFTWARE): - adjustment -= 2 - reasons.append("camera firmware in Software field") - - summary.trust_adjustment = adjustment - summary.trust_reason = "; ".join(reasons) if reasons else "no EXIF metadata found" - - logger.info(f"EXIF extracted: make={summary.make}, model={summary.model}, " - f"adjustment={adjustment} ({summary.trust_reason})") - return summary diff --git a/file_handler.py b/file_handler.py deleted file mode 100644 index dc88cbed4ecce474c2eb7b5d543bdd8ae9124717..0000000000000000000000000000000000000000 --- a/file_handler.py +++ /dev/null @@ -1,96 +0,0 @@ -from __future__ import annotations - -import io -import os -import tempfile -from typing import Iterable - -from fastapi import HTTPException, UploadFile, status - -from config import settings - -IMAGE_MAGIC_BYTES: dict[bytes, str] = { - b"\xff\xd8\xff": "image/jpeg", - b"\x89PNG\r\n\x1a\n": "image/png", - b"RIFF": "image/webp", # partial; WEBP has 'RIFF....WEBP' -} - - -def _detect_mime_by_magic(head: bytes) -> str | None: - for sig, mime in IMAGE_MAGIC_BYTES.items(): - if head.startswith(sig): - if mime == "image/webp" and b"WEBP" not in head[:16]: - continue - return mime - return None - - -async def read_upload_bytes( - file: UploadFile, - allowed_mimes: Iterable[str], - max_size_mb: int, -) -> tuple[bytes, str]: - """Read an UploadFile into memory after validating type and size. - Returns (raw_bytes, detected_mime). Raises HTTPException on failure. - """ - data = await file.read() - size_mb = len(data) / (1024 * 1024) - if size_mb > max_size_mb: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File too large ({size_mb:.1f} MB > {max_size_mb} MB)", - ) - - mime = _detect_mime_by_magic(data[:16]) or (file.content_type or "") - if mime not in allowed_mimes: - raise HTTPException( - status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, - detail=f"Unsupported type '{mime}'. Allowed: {list(allowed_mimes)}", - ) - return data, mime - - -def bytes_to_buffer(data: bytes) -> io.BytesIO: - return io.BytesIO(data) - - -async def save_upload_to_tempfile( - file: UploadFile, - allowed_mimes: Iterable[str], - max_size_mb: int, - suffix: str = ".mp4", -) -> tuple[str, str]: - """Stream an UploadFile to a temp file on disk. Returns (path, mime). - MIME is taken from the client's content_type (no magic-byte check for videos). - Caller is responsible for deleting the temp file. - """ - mime = (file.content_type or "").lower() - if mime not in allowed_mimes: - raise HTTPException( - status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, - detail=f"Unsupported type '{mime}'. Allowed: {list(allowed_mimes)}", - ) - - max_bytes = max_size_mb * 1024 * 1024 - fd, path = tempfile.mkstemp(suffix=suffix, prefix="ds_vid_") - written = 0 - try: - with os.fdopen(fd, "wb") as out: - while True: - chunk = await file.read(1024 * 1024) - if not chunk: - break - written += len(chunk) - if written > max_bytes: - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File too large (> {max_size_mb} MB)", - ) - out.write(chunk) - except Exception: - try: - os.unlink(path) - except OSError: - pass - raise - return path, mime diff --git a/generate_colab_nb.py b/generate_colab_nb.py deleted file mode 100644 index 9e1828737351e68461499d7786945060fc25fea9..0000000000000000000000000000000000000000 --- a/generate_colab_nb.py +++ /dev/null @@ -1,213 +0,0 @@ -import nbformat as nbf -import os - -nb = nbf.v4.new_notebook() - -text = """\ -# DeepShield: FaceForensics++ ViT Training -Run this entirely in Google Colab. -**Before running**: -1. Go to `Runtime` -> `Change runtime type` -> select **T4 GPU**. -2. Run the cells below sequentially. -""" - -code_install = """\ -!pip install timm transformers datasets accelerate evaluate opencv-python -""" - -code_ffpp = """\ -# We create the download script inside the Colab environment -download_script = '''#!/usr/bin/env python -import argparse -import os -import urllib.request -import tempfile -import time -import sys -import json -from tqdm import tqdm -from os.path import join - -FILELIST_URL = 'misc/filelist.json' -DEEPFEAKES_DETECTION_URL = 'misc/deepfake_detection_filenames.json' -DEEPFAKES_MODEL_NAMES = ['decoder_A.h5', 'decoder_B.h5', 'encoder.h5',] -DATASETS = { - 'original': 'original_sequences/youtube', - 'Deepfakes': 'manipulated_sequences/Deepfakes', - 'Face2Face': 'manipulated_sequences/Face2Face', - 'FaceShifter': 'manipulated_sequences/FaceShifter', - 'FaceSwap': 'manipulated_sequences/FaceSwap', - 'NeuralTextures': 'manipulated_sequences/NeuralTextures' -} -ALL_DATASETS = ['original', 'Deepfakes', 'Face2Face', 'FaceShifter', 'FaceSwap', 'NeuralTextures'] -COMPRESSION = ['raw', 'c23', 'c40'] -TYPE = ['videos'] - -def download_file(url, out_file): - os.makedirs(os.path.dirname(out_file), exist_ok=True) - if not os.path.isfile(out_file): - urllib.request.urlretrieve(url, out_file) - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('output_path', type=str) - parser.add_argument('-d', '--dataset', type=str, default='all') - parser.add_argument('-c', '--compression', type=str, default='c40') - parser.add_argument('-t', '--type', type=str, default='videos') - parser.add_argument('-n', '--num_videos', type=int, default=50) # Small amount for tutorial - args = parser.parse_args() - - base_url = 'http://kaldir.vc.in.tum.de/faceforensics/v3/' - - datasets = [args.dataset] if args.dataset != 'all' else ALL_DATASETS - for dataset in datasets: - dataset_path = DATASETS[dataset] - print(f'Downloading {args.compression} of {dataset}') - - file_pairs = json.loads(urllib.request.urlopen(base_url + FILELIST_URL).read().decode("utf-8")) - filelist = [] - if 'original' in dataset_path: - for pair in file_pairs: - filelist += pair - else: - for pair in file_pairs: - filelist.append('_'.join(pair)) - filelist.append('_'.join(pair[::-1])) - - filelist = filelist[:args.num_videos] - dataset_videos_url = base_url + f'{dataset_path}/{args.compression}/{args.type}/' - dataset_output_path = join(args.output_path, dataset_path, args.compression, args.type) - - for filename in tqdm(filelist): - download_file(dataset_videos_url + filename + ".mp4", join(dataset_output_path, filename + ".mp4")) - -if __name__ == "__main__": - main() -''' - -with open("download_ffpp.py", "w") as f: - f.write(download_script) - -!python download_ffpp.py ./data -d all -c c40 -t videos -n 50 -""" - -code_extract = """\ -import cv2 -import os -import glob -from tqdm import tqdm - -def extract_frames(video_folder, output_folder, label, max_frames=4): - os.makedirs(output_folder, exist_ok=True) - videos = glob.glob(os.path.join(video_folder, "*.mp4")) - - for vid_path in tqdm(videos, desc=f"Extracting {label}"): - vid_name = os.path.basename(vid_path).replace('.mp4','') - cap = cv2.VideoCapture(vid_path) - count = 0 - while cap.isOpened() and count < max_frames: - ret, frame = cap.read() - if not ret: break - frame = cv2.resize(frame, (224, 224)) - out_path = os.path.join(output_folder, f"{vid_name}_f{count}.jpg") - cv2.imwrite(out_path, frame) - count += 1 - cap.release() - -# Extract Real -extract_frames('./data/original_sequences/youtube/c40/videos', './dataset/real', 'real') - -# Extract Fakes -fakes = ['Deepfakes', 'Face2Face', 'FaceSwap', 'NeuralTextures'] -for f in fakes: - extract_frames(f'./data/manipulated_sequences/{f}/c40/videos', './dataset/fake', 'fake') -""" - -code_train = """\ -import numpy as np -from datasets import load_dataset -from transformers import ViTImageProcessor, ViTForImageClassification, TrainingArguments, Trainer -import torch - -# 1. Load Dataset -dataset = load_dataset('imagefolder', data_dir='./dataset') -# Split into train/validation -dataset = dataset['train'].train_test_split(test_size=0.1) - -# 2. Preprocessor -model_name_or_path = 'google/vit-base-patch16-224-in21k' -processor = ViTImageProcessor.from_pretrained(model_name_or_path) - -def transform(example_batch): - # Take a list of PIL images and turn them to pixel values - inputs = processor([x.convert("RGB") for x in example_batch['image']], return_tensors='pt') - inputs['labels'] = example_batch['label'] - return inputs - -prepared_ds = dataset.with_transform(transform) - -def collate_fn(batch): - return { - 'pixel_values': torch.stack([x['pixel_values'] for x in batch]), - 'labels': torch.tensor([x['labels'] for x in batch]) - } - -# 3. Load Model -labels = dataset['train'].features['label'].names -model = ViTForImageClassification.from_pretrained( - model_name_or_path, - num_labels=len(labels), - id2label={str(i): c for i, c in enumerate(labels)}, - label2id={c: str(i) for i, c in enumerate(labels)} -) - -training_args = TrainingArguments( - output_dir="./vit-deepshield", - per_device_train_batch_size=16, - eval_strategy="steps", - num_train_epochs=3, - fp16=True, # Mixed precision for speed - save_steps=100, - eval_steps=100, - logging_steps=10, - learning_rate=2e-4, - save_total_limit=2, - remove_unused_columns=False, - push_to_hub=False, - load_best_model_at_end=True, -) - -import evaluate -metric = evaluate.load("accuracy") -def compute_metrics(p): - return metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids) - -trainer = Trainer( - model=model, - args=training_args, - data_collator=collate_fn, - compute_metrics=compute_metrics, - train_dataset=prepared_ds["train"], - eval_dataset=prepared_ds["test"], -) - -# 4. Train -train_results = trainer.train() -trainer.save_model("deepshield_vit_model") -processor.save_pretrained("deepshield_vit_model") -trainer.log_metrics("train", train_results.metrics) -trainer.save_metrics("train", train_results.metrics) -trainer.save_state() -print("Training Complete! The model is saved to ./deepshield_vit_model") -""" - -nb['cells'] = [ - nbf.v4.new_markdown_cell(text), - nbf.v4.new_code_cell(code_install), - nbf.v4.new_code_cell(code_ffpp), - nbf.v4.new_code_cell(code_extract), - nbf.v4.new_code_cell(code_train) -] - -with open(r'c:\Users\athar\Desktop\minor2\backend\training\Colab_ViT_Training.ipynb', 'w', encoding='utf-8') as f: - nbf.write(nb, f) diff --git a/heatmap_generator.py b/heatmap_generator.py deleted file mode 100644 index 7f62e4d2b740d5ea02600380cd5ea4425950c577..0000000000000000000000000000000000000000 --- a/heatmap_generator.py +++ /dev/null @@ -1,164 +0,0 @@ -from __future__ import annotations - -import base64 -import io -from typing import Optional - -import cv2 -import numpy as np -import torch -from loguru import logger -from PIL import Image -from pytorch_grad_cam import GradCAMPlusPlus -from pytorch_grad_cam.utils.image import show_cam_on_image -from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget - -from config import settings -from models.model_loader import get_model_loader - - -class _HFLogitsWrapper(torch.nn.Module): - """Wrap a HuggingFace image classification model so forward() returns logits - as a plain tensor (pytorch_grad_cam expects tensor outputs, not dicts/dataclasses). - """ - - def __init__(self, model: torch.nn.Module) -> None: - super().__init__() - self.model = model - - def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: # type: ignore[override] - return self.model(pixel_values=pixel_values).logits - - -def _vit_reshape_transform(tensor: torch.Tensor, height: int = 14, width: int = 14) -> torch.Tensor: - """Grad-CAM expects (B, C, H, W); ViT hidden states are (B, 1+H*W, C). - Drop the CLS token and reshape tokens into a spatial grid. - """ - result = tensor[:, 1:, :] - b, n, c = result.shape - result = result.reshape(b, height, width, c) - result = result.permute(0, 3, 1, 2) # (B, C, H, W) - return result - - -def _preprocess_for_cam(pil_img: Image.Image, processor) -> tuple[torch.Tensor, np.ndarray]: - """Return (input_tensor, rgb_float_224) where rgb_float_224 is a (H,W,3) float - array in [0,1] matching the model input geometry — needed for overlaying. - """ - inputs = processor(images=pil_img, return_tensors="pt") - input_tensor = inputs["pixel_values"].to(settings.DEVICE) - - size = getattr(processor, "size", {"height": 224, "width": 224}) - h = size.get("height", 224) if isinstance(size, dict) else 224 - w = size.get("width", 224) if isinstance(size, dict) else 224 - - resized = pil_img.resize((w, h), Image.BILINEAR) - rgb = np.array(resized).astype(np.float32) / 255.0 # (H,W,3) in [0,1] - return input_tensor, rgb - - -def _encode_overlay_to_base64(overlay: np.ndarray) -> str: - """Encode a uint8 (H,W,3) RGB overlay to a base64 data-URL PNG.""" - buf = io.BytesIO() - Image.fromarray(overlay).save(buf, format="PNG") - b64 = base64.b64encode(buf.getvalue()).decode("ascii") - return f"data:image/png;base64,{b64}" - - -def _compute_gradcam_pp( - pil_img: Image.Image, - target_class_idx: Optional[int] = None, -) -> tuple[np.ndarray, np.ndarray]: - """Compute Grad-CAM++ averaged across the last 3 ViT encoder layers. - Returns (grayscale_cam, rgb_float) where grayscale_cam is (H,W) in [0,1]. - """ - loader = get_model_loader() - model, processor = loader.load_image_model() - - model.eval() - for p in model.parameters(): - p.requires_grad_(True) - - input_tensor, rgb_float = _preprocess_for_cam(pil_img, processor) - - grid = int(model.config.image_size / model.config.patch_size) - - # Average across last 3 ViT encoder layers for smoother heatmaps - num_layers = len(model.vit.encoder.layer) - last_n = min(3, num_layers) - target_layers = [ - model.vit.encoder.layer[-(i + 1)].layernorm_before - for i in range(last_n) - ] - - wrapped = _HFLogitsWrapper(model) - - targets = None - if target_class_idx is not None: - targets = [ClassifierOutputTarget(int(target_class_idx))] - - with GradCAMPlusPlus( - model=wrapped, - target_layers=target_layers, - reshape_transform=lambda t: _vit_reshape_transform(t, grid, grid), - ) as cam: - grayscale_cam = cam(input_tensor=input_tensor, targets=targets)[0] # (H,W) in [0,1] - - return grayscale_cam, rgb_float - - -def generate_heatmap_base64( - pil_img: Image.Image, - target_class_idx: Optional[int] = None, -) -> str: - """Produce a base64 data-URL PNG of the Grad-CAM++ overlay for the given image.""" - grayscale_cam, rgb_float = _compute_gradcam_pp(pil_img, target_class_idx) - overlay = show_cam_on_image(rgb_float, grayscale_cam, use_rgb=True) - logger.info(f"Heatmap generated ({overlay.shape[0]}x{overlay.shape[1]})") - return _encode_overlay_to_base64(overlay) - - -def generate_boxes_base64( - pil_img: Image.Image, - target_class_idx: Optional[int] = None, - top_k: int = 5, - threshold: float = 0.4, -) -> str: - """Produce bounding boxes around top-K connected components from Grad-CAM++ activation. - Renders colored boxes (red/yellow/orange by intensity) on the original image. - """ - grayscale_cam, rgb_float = _compute_gradcam_pp(pil_img, target_class_idx) - - h, w = rgb_float.shape[:2] - base_img = (rgb_float * 255).astype(np.uint8).copy() - - # Threshold the heatmap to find activated regions - binary = (grayscale_cam >= threshold).astype(np.uint8) * 255 - contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - - if not contours: - logger.info("No significant activation regions found for bounding boxes") - return _encode_overlay_to_base64(base_img) - - # Sort by area descending, take top_k - contours = sorted(contours, key=cv2.contourArea, reverse=True)[:top_k] - - # Color by mean activation intensity within each box - for cnt in contours: - x, y, bw, bh = cv2.boundingRect(cnt) - region_activation = grayscale_cam[y:y + bh, x:x + bw].mean() - - if region_activation >= 0.7: - color = (220, 40, 40) # red — high suspicion - elif region_activation >= 0.5: - color = (240, 140, 20) # orange — medium - else: - color = (230, 200, 40) # yellow — lower - - cv2.rectangle(base_img, (x, y), (x + bw, y + bh), color, 2) - label = f"{region_activation * 100:.0f}%" - cv2.putText(base_img, label, (x, max(y - 6, 12)), - cv2.FONT_HERSHEY_SIMPLEX, 0.4, color, 1, cv2.LINE_AA) - - logger.info(f"Bounding boxes generated: {len(contours)} regions") - return _encode_overlay_to_base64(base_img) diff --git a/image_service.py b/image_service.py deleted file mode 100644 index 799857aad686978ca8dc5289218a39ed4ddc328c..0000000000000000000000000000000000000000 --- a/image_service.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -import io -from dataclasses import dataclass -from typing import Tuple - -import torch -from loguru import logger -from PIL import Image - -from config import settings -from models.model_loader import get_model_loader - - -@dataclass -class ImageClassification: - label: str - confidence: float - all_scores: dict[str, float] - - -def load_image_from_bytes(data: bytes) -> Image.Image: - img = Image.open(io.BytesIO(data)) - if img.mode != "RGB": - img = img.convert("RGB") - return img - - -def classify_image(pil_img: Image.Image) -> ImageClassification: - """Run the ViT deepfake classifier on a PIL image.""" - loader = get_model_loader() - model, processor = loader.load_image_model() - - inputs = processor(images=pil_img, return_tensors="pt") - inputs = {k: v.to(settings.DEVICE) for k, v in inputs.items()} - - with torch.no_grad(): - outputs = model(**inputs) - logits = outputs.logits # (1, num_labels) - probs = torch.softmax(logits, dim=-1)[0] - - id2label: dict[int, str] = getattr(model.config, "id2label", {}) - all_scores = {id2label.get(i, str(i)): float(p.item()) for i, p in enumerate(probs)} - top_idx = int(torch.argmax(probs).item()) - top_label = id2label.get(top_idx, str(top_idx)) - top_conf = float(probs[top_idx].item()) - - logger.info(f"Image classify → {top_label} @ {top_conf:.3f}") - return ImageClassification(label=top_label, confidence=top_conf, all_scores=all_scores) - - -def preprocess_and_classify(raw_bytes: bytes) -> Tuple[Image.Image, ImageClassification]: - """Convenience: decode bytes → PIL → classify. Returns the PIL image too so - downstream steps (heatmap, artifact scan) can reuse it. - """ - pil = load_image_from_bytes(raw_bytes) - result = classify_image(pil) - return pil, result diff --git a/llm_explainer.py b/llm_explainer.py deleted file mode 100644 index d62e14d239944393e23c208ae480649907fcb0d2..0000000000000000000000000000000000000000 --- a/llm_explainer.py +++ /dev/null @@ -1,191 +0,0 @@ -"""LLM Explainability Card — Phase 12.3 - -Generates a plain-English summary paragraph + 3 key-signal bullets from the -full analysis payload. Supports Gemini (default) and OpenAI providers. -Results are cached per record_id to avoid re-spending tokens. -""" - -from __future__ import annotations - -import json -from abc import ABC, abstractmethod -from functools import lru_cache -from typing import Any - -from loguru import logger - -from config import settings -from schemas.common import LLMExplainabilitySummary - -# ── In-memory cache keyed by record_id ── -_cache: dict[str, LLMExplainabilitySummary] = {} - - -_PROMPT_TEMPLATE = """\ -You are DeepShield's explainability engine. Given the JSON analysis payload below, -write a concise, accessible summary for a non-technical user. - -**Output format (strict JSON only — no markdown fences):** -{{ - "paragraph": "<2-3 sentence plain-English summary of the verdict and key signals>", - "bullets": [ - "", - "", - "" - ] -}} - -Rules: -- Be factual. State what the analysis found, not what you speculate. -- Reference specific indicators (e.g. "GAN artifact score", "EXIF metadata", "sensationalism level"). -- If the verdict is "Likely Authentic", reassure the user and explain why. -- If the verdict is "Likely Manipulated" or "Suspicious", highlight the strongest evidence. -- Keep the paragraph under 60 words. Each bullet under 20 words. - -**Analysis payload:** -{payload_json} -""" - - -class _LLMProvider(ABC): - @abstractmethod - def generate(self, prompt: str) -> str: - """Send prompt to LLM and return raw text response.""" - - -class _GeminiProvider(_LLMProvider): - def __init__(self) -> None: - import google.generativeai as genai - genai.configure(api_key=settings.LLM_API_KEY) - self._model = genai.GenerativeModel(settings.LLM_MODEL) - - def generate(self, prompt: str) -> str: - response = self._model.generate_content(prompt) - return response.text - - -class _OpenAIProvider(_LLMProvider): - def __init__(self) -> None: - from openai import OpenAI - self._client = OpenAI(api_key=settings.LLM_API_KEY) - - def generate(self, prompt: str) -> str: - response = self._client.chat.completions.create( - model=settings.LLM_MODEL, - messages=[{"role": "user", "content": prompt}], - temperature=0.3, - max_tokens=300, - ) - return response.choices[0].message.content - - -@lru_cache(maxsize=1) -def _get_provider() -> _LLMProvider: - """Lazy-init the configured LLM provider (singleton).""" - provider_name = settings.LLM_PROVIDER.lower() - if provider_name == "openai": - return _OpenAIProvider() - return _GeminiProvider() - - -def _parse_llm_response(raw: str) -> tuple[str, list[str]]: - """Parse the LLM's JSON response into (paragraph, bullets). - Handles cases where the LLM wraps output in markdown fences. - """ - text = raw.strip() - # Strip markdown code fences if present - if text.startswith("```"): - lines = text.split("\n") - # Remove first and last fence lines - lines = [l for l in lines if not l.strip().startswith("```")] - text = "\n".join(lines).strip() - - parsed = json.loads(text) - paragraph = parsed.get("paragraph", "") - bullets = parsed.get("bullets", []) - if not isinstance(bullets, list): - bullets = [str(bullets)] - return paragraph, bullets[:3] - - -def generate_llm_summary( - payload: dict[str, Any], - record_id: str | None = None, -) -> LLMExplainabilitySummary: - """Generate an LLM-powered plain-English explanation for an analysis result. - - Args: - payload: The full analysis response dict (verdict, scores, indicators, etc.). - record_id: Optional cache key. If provided and cached, returns cached result. - - Returns: - LLMExplainabilitySummary with paragraph, bullets, and model info. - """ - # Check cache - if record_id and record_id in _cache: - logger.debug(f"LLM summary cache hit for record_id={record_id}") - cached = _cache[record_id] - cached.cached = True - return cached - - # Guard: no API key configured - if not settings.LLM_API_KEY: - logger.warning("LLM_API_KEY not set — using deterministic fallback summary") - - verdict_data = payload.get("verdict", {}) - label = verdict_data.get("label", "Unknown") - score = verdict_data.get("authenticity_score", 50) - - return LLMExplainabilitySummary( - paragraph=f"The DeepShield AI engine has analyzed this media and determined it is '{label}' with an authenticity score of {score}/100. We arrived at this conclusion by passing the file through our deepfake detection algorithms, artifact scanners, and metadata analyzers.", - bullets=[ - f"Overall Authenticity Score: {score}/100", - f"Primary Verdict: {label}", - "Note: Configure an LLM API key for deeper contextual analysis." - ], - model_used="static-fallback", - ) - - # Strip heavy base64 fields to reduce token usage - slim_payload = {k: v for k, v in payload.items() - if k not in ("explainability",)} - # Include explainability but strip base64 images - if "explainability" in payload and isinstance(payload["explainability"], dict): - expl = {k: v for k, v in payload["explainability"].items() - if not k.endswith("_base64")} - slim_payload["explainability"] = expl - - prompt = _PROMPT_TEMPLATE.format(payload_json=json.dumps(slim_payload, indent=2, default=str)) - - try: - provider = _get_provider() - raw_response = provider.generate(prompt) - paragraph, bullets = _parse_llm_response(raw_response) - - summary = LLMExplainabilitySummary( - paragraph=paragraph, - bullets=bullets, - model_used=f"{settings.LLM_PROVIDER}/{settings.LLM_MODEL}", - ) - - # Cache result - if record_id: - _cache[record_id] = summary - - logger.info(f"LLM summary generated via {settings.LLM_PROVIDER}/{settings.LLM_MODEL}") - return summary - - except json.JSONDecodeError as e: - logger.error(f"LLM returned unparseable JSON: {e}") - return LLMExplainabilitySummary( - paragraph="Analysis complete. See the detailed indicators below for specifics.", - bullets=["LLM explanation could not be parsed"], - model_used=f"{settings.LLM_PROVIDER}/{settings.LLM_MODEL}", - ) - except Exception as e: - logger.error(f"LLM explainer failed: {e}") - return LLMExplainabilitySummary( - paragraph="Analysis complete. See the detailed indicators below for specifics.", - bullets=["LLM explanation temporarily unavailable"], - model_used="error", - ) diff --git a/logs/deepshield.log b/logs/deepshield.log new file mode 100644 index 0000000000000000000000000000000000000000..391295e63755b5303db9a7967e8305ea9342f884 --- /dev/null +++ b/logs/deepshield.log @@ -0,0 +1,949 @@ +2026-04-22 18:24:59.601 | INFO | main:lifespan:83 - Starting DeepShield backend +2026-04-22 18:24:59.655 | INFO | main:lifespan:85 - Database initialized +2026-04-22 18:24:59.656 | INFO | models.model_loader:load_image_model:43 - Loading image model: prithivMLmods/Deep-Fake-Detector-v2-Model +2026-04-22 18:25:06.201 | INFO | models.model_loader:load_image_model:51 - Image model loaded +2026-04-22 18:25:06.206 | INFO | services.report_service:cleanup_expired:151 - Cleaned up 1 expired reports +2026-04-22 18:26:20.263 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-22 18:26:22.700 | INFO | services.efficientnet_service:__init__:97 - EfficientNetDetector ready: EfficientNetAutoAttB4/DFDC on cpu | calibrator=no +2026-04-22 18:26:23.034 | INFO | services.image_service:classify_image:152 - Image classify (average_vit_eff) → Real | vit=0.078 ffpp=n/a eff=0.18335410952568054 → 0.131 +2026-04-22 18:26:28.349 | INFO | models.model_loader:load_face_detector:142 - Loading MediaPipe FaceMesh +2026-04-22 18:26:28.390 | INFO | models.model_loader:load_face_detector:150 - MediaPipe FaceMesh loaded +2026-04-22 18:26:29.238 | INFO | models.heatmap_generator:generate_heatmap_base64:186 - Heatmap generated (224x224) source=gradcam++ +2026-04-22 18:26:29.277 | INFO | services.ela_service:generate_ela_base64:60 - ELA map generated (256x256) +2026-04-22 18:26:30.141 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 5 regions +2026-04-22 18:26:30.327 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=None, model=None, adjustment=0 (no EXIF metadata found) +2026-04-22 18:26:30.347 | INFO | api.v1.analyze:analyze_image:214 - Saved AnalysisRecord id=19 score=13 verdict=Very Likely Fake +2026-04-22 18:26:30.349 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: No module named 'google.generativeai' +2026-04-22 18:26:30.349 | ERROR | services.vlm_breakdown:generate_vlm_breakdown:104 - VLM breakdown failed: No module named 'google.generativeai' +2026-04-22 18:27:58.805 | INFO | main:lifespan:93 - Shutting down DeepShield backend +2026-04-22 18:28:09.692 | INFO | main:lifespan:83 - Starting DeepShield backend +2026-04-22 18:28:09.698 | INFO | main:lifespan:85 - Database initialized +2026-04-22 18:28:09.698 | INFO | models.model_loader:load_image_model:43 - Loading image model: prithivMLmods/Deep-Fake-Detector-v2-Model +2026-04-22 18:28:11.556 | INFO | models.model_loader:load_image_model:51 - Image model loaded +2026-04-24 01:50:58.220 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-24 01:51:03.592 | INFO | services.efficientnet_service:__init__:97 - EfficientNetDetector ready: EfficientNetAutoAttB4/DFDC on cpu | calibrator=no +2026-04-24 01:51:03.887 | INFO | services.image_service:classify_image:152 - Image classify (vit_only) → Fake | vit=0.597 ffpp=n/a eff=n/a → 0.597 +2026-04-24 01:51:12.975 | INFO | models.model_loader:load_face_detector:142 - Loading MediaPipe FaceMesh +2026-04-24 01:51:13.089 | INFO | models.model_loader:load_face_detector:150 - MediaPipe FaceMesh loaded +2026-04-24 01:51:13.255 | INFO | models.heatmap_generator:generate_heatmap_base64:176 - EfficientNet heatmap skipped — no face detected +2026-04-24 01:51:13.320 | INFO | services.ela_service:generate_ela_base64:60 - ELA map generated (640x427) +2026-04-24 01:51:14.648 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 1 regions +2026-04-24 01:51:14.933 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=None, model=None, adjustment=0 (no EXIF metadata found) +2026-04-24 01:51:14.979 | INFO | api.v1.analyze:analyze_image:215 - Saved AnalysisRecord id=20 score=40 verdict=Likely Fake +2026-04-24 01:51:14.982 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: No module named 'google.generativeai' +2026-04-24 01:51:14.984 | ERROR | services.vlm_breakdown:generate_vlm_breakdown:104 - VLM breakdown failed: No module named 'google.generativeai' +2026-04-24 07:35:53.458 | INFO | models.model_loader:load_text_model:57 - Loading text model: jy46604790/Fake-News-Bert-Detect +2026-04-24 07:36:02.194 | INFO | models.model_loader:load_text_model:65 - Text model loaded +2026-04-24 07:36:03.057 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.999 fake_p=0.999 +2026-04-24 07:36:03.058 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 68 (High) excl=4 caps=3 cb=1 emo=1 +2026-04-24 07:36:03.061 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 3 found +2026-04-24 07:36:05.585 | WARNING | models.model_loader:load_spacy_nlp:98 - spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm +2026-04-24 07:36:06.959 | INFO | api.v1.analyze:analyze_text_endpoint:550 - Saved AnalysisRecord id=21 text score=15 verdict=Very Likely Fake +2026-04-24 07:36:08.561 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +Please retry in 51.884484839s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerMinute-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerDay-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +, retry_delay { + seconds: 51 +} +] +2026-04-24 07:36:41.979 | INFO | models.model_loader:load_image_model:43 - Loading image model: prithivMLmods/Deep-Fake-Detector-v2-Model +2026-04-24 07:36:47.524 | INFO | models.model_loader:load_image_model:51 - Image model loaded +2026-04-24 07:36:48.484 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-24 07:36:49.759 | INFO | services.efficientnet_service:__init__:97 - EfficientNetDetector ready: EfficientNetAutoAttB4/DFDC on cpu | calibrator=no +2026-04-24 07:36:49.848 | INFO | services.image_service:classify_image:152 - Image classify (vit_only) → Fake | vit=0.521 ffpp=n/a eff=n/a → 0.521 +2026-04-24 07:36:51.638 | INFO | models.model_loader:load_face_detector:142 - Loading MediaPipe FaceMesh +2026-04-24 07:36:51.638 | WARNING | services.artifact_detector:detect_face_based_artifacts:213 - Face-based artifact detection failed: module 'mediapipe' has no attribute 'solutions' +2026-04-24 07:36:51.649 | INFO | models.heatmap_generator:generate_heatmap_base64:176 - EfficientNet heatmap skipped — no face detected +2026-04-24 07:36:51.696 | INFO | services.ela_service:generate_ela_base64:60 - ELA map generated (512x512) +2026-04-24 07:36:52.470 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 5 regions +2026-04-24 07:36:52.519 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=None, model=None, adjustment=0 (no EXIF metadata found) +2026-04-24 07:36:52.542 | INFO | api.v1.analyze:analyze_image:215 - Saved AnalysisRecord id=22 score=48 verdict=Possibly Manipulated +2026-04-24 07:36:53.674 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +Please retry in 6.748563195s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerMinute-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerDay-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +, retry_delay { + seconds: 6 +} +] +2026-04-24 07:36:54.760 | ERROR | services.vlm_breakdown:generate_vlm_breakdown:104 - VLM breakdown failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +Please retry in 5.653927512s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerMinute-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerDay-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +, retry_delay { + seconds: 5 +} +] +2026-04-24 15:16:36.138 | INFO | models.model_loader:load_text_model:57 - Loading text model: jy46604790/Fake-News-Bert-Detect +2026-04-24 15:16:43.946 | INFO | models.model_loader:load_text_model:65 - Text model loaded +2026-04-24 15:16:44.719 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.998 fake_p=0.998 +2026-04-24 15:16:44.721 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 67 (High) excl=3 caps=2 cb=1 emo=1 +2026-04-24 15:16:44.723 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 3 found +2026-04-24 15:16:45.864 | WARNING | models.model_loader:load_spacy_nlp:98 - spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm +2026-04-24 15:16:47.113 | INFO | api.v1.analyze:analyze_text_endpoint:549 - Saved AnalysisRecord id=23 text score=15 verdict=Very Likely Fake +2026-04-24 15:16:48.348 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +Please retry in 12.294521515s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerMinute-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerDay-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +, retry_delay { + seconds: 12 +} +] +2026-04-24 15:16:48.553 | INFO | models.model_loader:load_image_model:43 - Loading image model: prithivMLmods/Deep-Fake-Detector-v2-Model +2026-04-24 15:16:50.111 | INFO | models.model_loader:load_image_model:51 - Image model loaded +2026-04-24 15:16:51.265 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-24 15:16:52.685 | INFO | services.efficientnet_service:__init__:97 - EfficientNetDetector ready: EfficientNetAutoAttB4/DFDC on cpu | calibrator=no +2026-04-24 15:16:52.723 | INFO | services.image_service:classify_image:152 - Image classify (vit_only) → Fake | vit=0.517 ffpp=n/a eff=n/a → 0.517 +2026-04-24 15:16:52.735 | INFO | models.model_loader:load_face_detector:142 - Loading MediaPipe FaceMesh +2026-04-24 15:16:54.934 | WARNING | services.artifact_detector:detect_face_based_artifacts:211 - Face-based artifact detection failed: module 'mediapipe' has no attribute 'solutions' +2026-04-24 15:16:54.949 | INFO | models.heatmap_generator:generate_heatmap_base64:176 - EfficientNet heatmap skipped — no face detected +2026-04-24 15:16:54.965 | INFO | services.ela_service:generate_ela_base64:59 - ELA map generated (256x256) +2026-04-24 15:16:55.916 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 5 regions +2026-04-24 15:16:55.975 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=None, model=None, adjustment=0 (no EXIF metadata found) +2026-04-24 15:16:55.989 | INFO | api.v1.analyze:analyze_image:214 - Saved AnalysisRecord id=24 score=48 verdict=Possibly Manipulated +2026-04-24 15:16:56.236 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +Please retry in 4.477916448s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerMinute-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerDay-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +, retry_delay { + seconds: 4 +} +] +2026-04-24 15:16:57.419 | ERROR | services.vlm_breakdown:generate_vlm_breakdown:104 - VLM breakdown failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +Please retry in 3.282459328s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerDay-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerMinute-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +, retry_delay { + seconds: 3 +} +] +2026-04-24 15:16:57.445 | INFO | models.model_loader:load_ocr_engine:130 - Loading EasyOCR reader (langs: ['en', 'hi']) +2026-04-24 15:17:27.399 | INFO | models.model_loader:load_ocr_engine:136 - EasyOCR loaded +2026-04-24 15:17:27.870 | INFO | services.screenshot_service:run_ocr:48 - OCR extracted 0 text regions +2026-04-24 15:17:27.881 | INFO | api.v1.analyze:analyze_screenshot_endpoint:726 - Saved AnalysisRecord id=25 screenshot score=50 verdict=Possibly Manipulated +2026-04-24 15:17:28.066 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +Please retry in 32.593323033s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerDay-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerMinute-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +, retry_delay { + seconds: 32 +} +] +2026-04-24 15:17:54.819 | INFO | models.model_loader:load_text_model:57 - Loading text model: jy46604790/Fake-News-Bert-Detect +2026-04-24 15:18:00.795 | INFO | models.model_loader:load_text_model:65 - Text model loaded +2026-04-24 15:18:00.888 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.998 fake_p=0.998 +2026-04-24 15:18:00.889 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 67 (High) excl=3 caps=2 cb=1 emo=1 +2026-04-24 15:18:00.891 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 3 found +2026-04-24 15:18:01.659 | WARNING | models.model_loader:load_spacy_nlp:98 - spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm +2026-04-24 15:18:02.878 | INFO | api.v1.analyze:analyze_text_endpoint:549 - Saved AnalysisRecord id=26 text score=15 verdict=Very Likely Fake +2026-04-24 15:18:03.994 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_input_token_count, limit: 0, model: gemini-2.5-pro +Please retry in 56.638939454s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerMinute-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_input_token_count" + quota_id: "GenerateContentInputTokensPerModelPerDay-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-pro" + } + quota_dimensions { + key: "location" + value: "global" + } +} +, retry_delay { + seconds: 56 +} +] +2026-04-24 15:20:38.285 | INFO | models.model_loader:load_text_model:57 - Loading text model: jy46604790/Fake-News-Bert-Detect +2026-04-24 15:20:43.929 | INFO | models.model_loader:load_text_model:65 - Text model loaded +2026-04-24 15:20:44.034 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.998 fake_p=0.998 +2026-04-24 15:20:44.035 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 67 (High) excl=3 caps=2 cb=1 emo=1 +2026-04-24 15:20:44.037 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 3 found +2026-04-24 15:20:44.806 | WARNING | models.model_loader:load_spacy_nlp:98 - spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm +2026-04-24 15:20:46.001 | INFO | api.v1.analyze:analyze_text_endpoint:549 - Saved AnalysisRecord id=27 text score=15 verdict=Very Likely Fake +2026-04-24 15:20:56.376 | INFO | services.llm_explainer:generate_llm_summary:175 - LLM summary generated via gemini/gemini-2.5-flash +2026-04-24 15:33:56.592 | INFO | api.v1.auth:register:33 - Registered user id=3 email=***@example.com +2026-04-24 15:33:57.227 | INFO | api.v1.auth:login:42 - Login user id=3 email=***@example.com +2026-04-24 15:33:57.553 | INFO | models.model_loader:load_text_model:57 - Loading text model: jy46604790/Fake-News-Bert-Detect +2026-04-24 15:34:06.986 | INFO | models.model_loader:load_text_model:65 - Text model loaded +2026-04-24 15:34:07.731 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.997 fake_p=0.997 +2026-04-24 15:34:07.733 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:34:07.736 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:34:09.017 | WARNING | models.model_loader:load_spacy_nlp:98 - spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm +2026-04-24 15:34:10.285 | INFO | api.v1.analyze:analyze_text_endpoint:549 - Saved AnalysisRecord id=28 text score=30 verdict=Likely Fake +2026-04-24 15:34:41.718 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, model: gemini-2.5-flash +Please retry in 19.188761533s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-flash" + } + quota_dimensions { + key: "location" + value: "global" + } + quota_value: 5 +} +, retry_delay { + seconds: 19 +} +] +2026-04-24 15:34:41.788 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.997 fake_p=0.997 +2026-04-24 15:34:41.788 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 76 (High) excl=3 caps=2 cb=1 emo=3 +2026-04-24 15:34:41.789 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:34:41.791 | WARNING | models.model_loader:load_spacy_nlp:98 - spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm +2026-04-24 15:34:43.147 | INFO | api.v1.analyze:analyze_text_endpoint:549 - Saved AnalysisRecord id=29 text score=15 verdict=Very Likely Fake +2026-04-24 15:34:43.555 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, model: gemini-2.5-flash +Please retry in 17.333464233s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-flash" + } + quota_dimensions { + key: "location" + value: "global" + } + quota_value: 5 +} +, retry_delay { + seconds: 17 +} +] +2026-04-24 15:34:43.615 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.996 fake_p=0.996 +2026-04-24 15:34:43.616 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:34:43.616 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:34:43.618 | WARNING | models.model_loader:load_spacy_nlp:98 - spaCy model 'en_core_web_sm' not found. Run: python -m spacy download en_core_web_sm +2026-04-24 15:34:44.924 | INFO | api.v1.analyze:analyze_text_endpoint:549 - Saved AnalysisRecord id=30 text score=30 verdict=Likely Fake +2026-04-24 15:34:45.353 | ERROR | services.llm_explainer:generate_llm_summary:186 - LLM explainer failed: 429 You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. +* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, model: gemini-2.5-flash +Please retry in 15.553103918s. [links { + description: "Learn more about Gemini API quotas" + url: "https://ai.google.dev/gemini-api/docs/rate-limits" +} +, violations { + quota_metric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" + quota_id: "GenerateRequestsPerMinutePerProjectPerModel-FreeTier" + quota_dimensions { + key: "model" + value: "gemini-2.5-flash" + } + quota_dimensions { + key: "location" + value: "global" + } + quota_value: 5 +} +, retry_delay { + seconds: 15 +} +] +2026-04-24 15:43:27.438 | INFO | api.v1.auth:register:33 - Registered user id=4 email=***@example.com +2026-04-24 15:43:27.463 | INFO | models.model_loader:load_text_model:57 - Loading text model: jy46604790/Fake-News-Bert-Detect +2026-04-24 15:43:33.684 | INFO | models.model_loader:load_text_model:65 - Text model loaded +2026-04-24 15:43:33.796 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.991 fake_p=0.991 +2026-04-24 15:43:33.797 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:43:33.799 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:43:35.106 | INFO | models.model_loader:load_spacy_nlp:96 - spaCy en_core_web_sm loaded +2026-04-24 15:43:35.120 | INFO | services.text_service:extract_entities:253 - NER extracted 3 entities: ['India', 'Elon Musk', 'New Delhi'] +2026-04-24 15:43:36.284 | INFO | api.v1.analyze:analyze_text_endpoint:550 - Saved AnalysisRecord id=31 text score=31 verdict=Likely Fake +2026-04-24 15:43:36.352 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.991 fake_p=0.991 +2026-04-24 15:43:36.352 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:43:36.353 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:43:36.370 | INFO | services.text_service:extract_entities:253 - NER extracted 3 entities: ['India', 'Elon Musk', 'New Delhi'] +2026-04-24 15:43:37.567 | INFO | api.v1.analyze:analyze_text_endpoint:550 - Saved AnalysisRecord id=32 text score=31 verdict=Likely Fake +2026-04-24 15:43:47.549 | INFO | services.llm_explainer:generate_llm_summary:207 - LLM summary generated via gemini/gemini-2.5-flash +2026-04-24 15:43:47.614 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.991 fake_p=0.991 +2026-04-24 15:43:47.614 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:43:47.615 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:43:47.630 | INFO | services.text_service:extract_entities:253 - NER extracted 3 entities: ['India', 'Elon Musk', 'New Delhi'] +2026-04-24 15:43:49.134 | INFO | api.v1.analyze:analyze_text_endpoint:550 - Saved AnalysisRecord id=33 text score=31 verdict=Likely Fake +2026-04-24 15:44:11.346 | WARNING | services.llm_explainer:mark_rate_limited:42 - LLM rate-limited — pausing all LLM calls for 300s +2026-04-24 15:44:11.346 | WARNING | services.llm_explainer:generate_llm_summary:220 - LLM quota hit (ResourceExhausted) — circuit open for 300s +2026-04-24 15:44:11.352 | WARNING | services.llm_explainer:mark_rate_limited:42 - LLM rate-limited — pausing all LLM calls for 5s +2026-04-24 15:44:11.404 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.999 fake_p=0.999 +2026-04-24 15:44:11.404 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:44:11.405 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:44:12.724 | INFO | api.v1.analyze:analyze_text_endpoint:550 - Saved AnalysisRecord id=34 text score=30 verdict=Likely Fake +2026-04-24 15:57:39.916 | INFO | api.v1.auth:register:33 - Registered user id=5 email=***@example.com +2026-04-24 15:57:39.958 | INFO | models.model_loader:load_text_model:57 - Loading text model: jy46604790/Fake-News-Bert-Detect +2026-04-24 15:57:46.475 | INFO | models.model_loader:load_text_model:65 - Text model loaded +2026-04-24 15:57:46.582 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.999 fake_p=0.999 +2026-04-24 15:57:46.584 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:57:46.586 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:57:47.954 | INFO | models.model_loader:load_spacy_nlp:96 - spaCy en_core_web_sm loaded +2026-04-24 15:57:49.166 | INFO | api.v1.analyze:analyze_text_endpoint:555 - Saved AnalysisRecord id=35 text score=30 verdict=Likely Fake +2026-04-24 15:57:58.130 | INFO | services.llm_explainer:generate_llm_summary:271 - LLM summary generated via gemini/gemini-2.5-flash +2026-04-24 15:57:58.196 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.999 fake_p=0.999 +2026-04-24 15:57:58.197 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:57:58.197 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:57:59.705 | INFO | api.v1.analyze:analyze_text_endpoint:555 - Saved AnalysisRecord id=36 text score=30 verdict=Likely Fake +2026-04-24 15:58:02.948 | ERROR | services.llm_explainer:generate_llm_summary:287 - LLM explainer failed: 503 UNAVAILABLE. {'error': {'code': 503, 'message': 'This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.', 'status': 'UNAVAILABLE'}} +2026-04-24 15:58:03.008 | INFO | services.text_service:classify_text:159 - Text classify [en] → LABEL_0 @ 0.999 fake_p=0.999 +2026-04-24 15:58:03.008 | INFO | services.text_service:score_sensationalism:193 - Sensationalism → 0 (Low) excl=0 caps=0 cb=0 emo=0 +2026-04-24 15:58:03.009 | INFO | services.text_service:detect_manipulation_indicators:213 - Manipulation indicators → 0 found +2026-04-24 15:58:04.488 | INFO | api.v1.analyze:analyze_text_endpoint:555 - Saved AnalysisRecord id=37 text score=30 verdict=Likely Fake +2026-04-24 15:59:52.694 | INFO | services.llm_explainer:_get_provider:176 - LLM chain initialized: gemini/gemini-2.5-flash → groq/llama-3.3-70b-versatile +2026-04-24 15:59:52.695 | INFO | services.llm_explainer:generate:161 - gemini/gemini-2.5-flash quota hit — failing over to groq/llama-3.3-70b-versatile +2026-04-24 23:15:36.409 | INFO | main:lifespan:108 - Starting DeepShield backend +2026-04-24 23:15:36.470 | INFO | main:lifespan:110 - Database initialized +2026-04-24 23:15:36.470 | INFO | models.model_loader:load_image_model:43 - Loading image model: prithivMLmods/Deep-Fake-Detector-v2-Model +2026-04-24 23:15:46.404 | INFO | models.model_loader:load_image_model:51 - Image model loaded +2026-04-24 23:15:57.188 | INFO | api.v1.analyze:analyze_image:118 - cache hit image sha=6de55b9fc5bd record=19 +2026-04-24 23:16:59.860 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-24 23:17:03.920 | INFO | services.efficientnet_service:__init__:97 - EfficientNetDetector ready: EfficientNetAutoAttB4/DFDC on cpu | calibrator=no +2026-04-24 23:17:04.519 | INFO | services.image_service:classify_image:152 - Image classify (average_vit_eff) → Real | vit=0.868 ffpp=n/a eff=0.03269108012318611 → 0.450 +2026-04-24 23:17:04.569 | INFO | models.model_loader:load_face_detector:142 - Loading MediaPipe FaceMesh +2026-04-24 23:17:13.315 | INFO | models.model_loader:load_face_detector:150 - MediaPipe FaceMesh loaded +2026-04-24 23:17:16.988 | INFO | models.heatmap_generator:generate_heatmap_base64:186 - Heatmap generated (224x224) source=gradcam++ +2026-04-24 23:17:17.131 | INFO | services.ela_service:generate_ela_base64:59 - ELA map generated (800x450) +2026-04-24 23:17:18.394 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 5 regions +2026-04-24 23:17:18.714 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=None, model=None, adjustment=0 (no EXIF metadata found) +2026-04-24 23:17:18.757 | INFO | api.v1.analyze:analyze_image:230 - Saved AnalysisRecord id=38 score=45 verdict=Possibly Manipulated +2026-04-24 23:29:04.622 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-24 23:29:05.312 | INFO | services.image_service:classify_image:152 - Image classify (average_vit_eff) → Fake | vit=0.767 ffpp=n/a eff=0.36121347546577454 → 0.564 +2026-04-24 23:29:06.604 | INFO | models.heatmap_generator:generate_heatmap_base64:186 - Heatmap generated (224x224) source=gradcam++ +2026-04-24 23:29:10.091 | INFO | services.ela_service:generate_ela_base64:59 - ELA map generated (2393x4096) +2026-04-24 23:29:11.326 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 5 regions +2026-04-24 23:29:11.344 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=None, model=None, adjustment=0 (no EXIF metadata found) +2026-04-24 23:29:11.436 | INFO | api.v1.analyze:analyze_image:230 - Saved AnalysisRecord id=39 score=44 verdict=Possibly Manipulated +2026-04-24 23:30:58.303 | ERROR | api.v1.report:generate:51 - Report generation failed: int() argument must be a string, a bytes-like object or a real number, not 'NoneType' +Traceback (most recent call last): + + File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\threading.py", line 1002, in _bootstrap + self._bootstrap_inner() + │ └ + └ + File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\threading.py", line 1045, in _bootstrap_inner + self.run() + │ └ + └ + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\anyio\_backends\_asyncio.py", line 1002, in run + result = context.run(func, *args) + │ │ │ └ () + │ │ └ functools.partial(, db=... + │ └ + └ <_contextvars.Context object at 0x000001A70D16CD40> + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\slowapi\extension.py", line 766, in sync_wrapper + response = func(*args, **kwargs) + │ │ └ {'db': , 'user': None, 'analysis_id': 39, 'request': + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\slowapi\extension.py", line 766, in sync_wrapper + response = func(*args, **kwargs) + │ │ └ {'db': , 'user': None, 'analysis_id': 39, 'request': + +> File "C:\Users\athar\Desktop\minor2\backend\api\v1\report.py", line 49, in generate + path = generate_report(record) + │ └ + └ + + File "C:\Users\athar\Desktop\minor2\backend\services\report_service.py", line 119, in generate_report + html_to_pdf(html, out_path) + │ │ └ WindowsPath('temp_reports/deepshield_39_c2b71295.pdf') + │ └ '\n\n\n \n DeepShield Analysis Report — c9f44067-528d-4e96-9365-2... + └ <function html_to_pdf at 0x000001A7011B9C60> + + File "C:\Users\athar\Desktop\minor2\backend\services\report_service.py", line 107, in html_to_pdf + result = pisa.CreatePDF(html, dest=f) + │ │ │ └ <_io.BufferedWriter name='temp_reports\\deepshield_39_c2b71295.pdf'> + │ │ └ '<!DOCTYPE html>\n<html>\n<head>\n <meta charset="utf-8" />\n <title>DeepShield Analysis Report — c9f44067-528d-4e96-9365-2... + │ └ <function pisaDocument at 0x000001A7011B9440> + └ <module 'xhtml2pdf.pisa' from 'C:\\Users\\athar\\Desktop\\minor2\\backend\\.venv\\Lib\\site-packages\\xhtml2pdf\\pisa.py'> + + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\xhtml2pdf\document.py", line 196, in pisaDocument + doc.build(context.story) + │ │ │ └ [PmlParagraph( + │ │ │ 'dir' + │ │ │ 'dir' + │ │ │ 'caseSensitive' + │ │ │ 'caseSensitive' + │ │ │ 'encoding' + │ │ │ 'encoding' + │ │ │ 'text' + │ │ │ 'text... + │ │ └ <xhtml2pdf.context.pisaContext object at 0x000001A703A22990> + │ └ <function BaseDocTemplate.build at 0x000001A77EFA8E00> + └ <xhtml2pdf.xhtml2pdf_reportlab.PmlBaseDoc object at 0x000001A703756C10> + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\reportlab\platypus\doctemplate.py", line 1083, in build + self.handle_flowable(flowables) + │ │ └ [PmlParagraph( + │ │ 'dir' + │ │ 'dir' + │ │ 'caseSensitive' + │ │ 'caseSensitive' + │ │ 'encoding' + │ │ 'encoding' + │ │ 'text' + │ │ 'text... + │ └ <function BaseDocTemplate.handle_flowable at 0x000001A77EFA8B80> + └ <xhtml2pdf.xhtml2pdf_reportlab.PmlBaseDoc object at 0x000001A703756C10> + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\reportlab\platypus\doctemplate.py", line 932, in handle_flowable + if frame.add(f, canv, trySplit=self.allowSplitting): + │ │ │ │ │ └ 1 + │ │ │ │ └ <xhtml2pdf.xhtml2pdf_reportlab.PmlBaseDoc object at 0x000001A703756C10> + │ │ │ └ <reportlab.pdfgen.canvas.Canvas object at 0x000001A70D1DED50> + │ │ └ PmlTable( + │ │ rowHeights=[None], + │ │ colWidths=[4.93228346456693, 488.29606299212605], + │ │ [[(<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInF... + │ └ <function Frame._add at 0x000001A77EECDF80> + └ <reportlab.platypus.frames.Frame object at 0x000001A70344D6D0> + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\reportlab\platypus\frames.py", line 158, in _add + w, h = flowable.wrap(aW, h) + │ │ │ └ 751.1811023622049 + │ │ └ 493.228346456693 + │ └ <function PmlTable.wrap at 0x000001A7011719E0> + └ PmlTable( + rowHeights=[None], + colWidths=[4.93228346456693, 488.29606299212605], + [[(<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInF... + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\xhtml2pdf\xhtml2pdf_reportlab.py", line 858, in wrap + return Table.wrap(self, availWidth, availHeight) + │ │ │ │ └ 751.1811023622049 + │ │ │ └ 493.228346456693 + │ │ └ PmlTable( + │ │ rowHeights=[None], + │ │ colWidths=[4.93228346456693, 488.29606299212605], + │ │ [[(<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInF... + │ └ <function Table.wrap at 0x000001A77EFAC400> + └ <class 'reportlab.platypus.tables.Table'> + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\reportlab\platypus\tables.py", line 1354, in wrap + self._calc(availWidth, availHeight) + │ │ │ └ 751.1811023622049 + │ │ └ 493.228346456693 + │ └ <function Table._calc at 0x000001A77EFAB600> + └ PmlTable( + rowHeights=[None], + colWidths=[4.93228346456693, 488.29606299212605], + [[(<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInF... + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\reportlab\platypus\tables.py", line 740, in _calc + self._calc_height(availHeight,availWidth,W=W) + │ │ │ │ └ None + │ │ │ └ 493.228346456693 + │ │ └ 751.1811023622049 + │ └ <function Table._calc_height at 0x000001A77EFAB560> + └ PmlTable( + rowHeights=[None], + colWidths=[4.93228346456693, 488.29606299212605], + [[(<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInF... + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\reportlab\platypus\tables.py", line 664, in _calc_height + dW,t = self._listCellGeom(v,w or self._listValueWidth(v),s) + │ │ │ │ │ │ │ └ <CellStyle '(0, 0)'> + │ │ │ │ │ │ └ (<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInFrame object at 0x000001A70D1F4950>,) + │ │ │ │ │ └ <function Table._listValueWidth at 0x000001A77EFAB380> + │ │ │ │ └ PmlTable( + │ │ │ │ rowHeights=[None], + │ │ │ │ colWidths=[4.93228346456693, 488.29606299212605], + │ │ │ │ [[(<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInF... + │ │ │ └ 4.93228346456693 + │ │ └ (<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInFrame object at 0x000001A70D1F4950>,) + │ └ <function PmlTable._listCellGeom at 0x000001A701171940> + └ PmlTable( + rowHeights=[None], + colWidths=[4.93228346456693, 488.29606299212605], + [[(<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInF... + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\xhtml2pdf\xhtml2pdf_reportlab.py", line 810, in _listCellGeom + return Table._listCellGeom(self, V, w, s, W=W, H=H, aH=aH) + │ │ │ │ │ │ │ │ └ 751.1811023622049 + │ │ │ │ │ │ │ └ None + │ │ │ │ │ │ └ None + │ │ │ │ │ └ <CellStyle '(0, 0)'> + │ │ │ │ └ 4.93228346456693 + │ │ │ └ (<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInFrame object at 0x000001A70D1F4950>,) + │ │ └ PmlTable( + │ │ rowHeights=[None], + │ │ colWidths=[4.93228346456693, 488.29606299212605], + │ │ [[(<xhtml2pdf.xhtml2pdf_reportlab.PmlKeepInF... + │ └ <function Table._listCellGeom at 0x000001A77EFAB2E0> + └ <class 'reportlab.platypus.tables.Table'> + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\reportlab\platypus\tables.py", line 490, in _listCellGeom + raise ValueError(f'{self.identity()}: flowable given negative availWidth={aW} == width={w} - leftPadding={s.leftPadding} - rightPadding={s.rightPadding}') + File "C:\Users\athar\Desktop\minor2\backend\.venv\Lib\site-packages\reportlab\platypus\tables.py", line 440, in identity + tallest = '(tallest row %d)' % int(max(rh)) + └ [None] + +TypeError: int() argument must be a string, a bytes-like object or a real number, not 'NoneType' +2026-04-24 23:44:20.465 | INFO | api.v1.auth:register:33 - Registered user id=6 email=***@gmail.com +2026-04-24 23:45:54.152 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-24 23:45:54.595 | INFO | services.image_service:classify_image:152 - Image classify (average_vit_eff) → Real | vit=0.668 ffpp=n/a eff=0.00913542602211237 → 0.339 +2026-04-24 23:45:55.772 | INFO | models.heatmap_generator:generate_heatmap_base64:186 - Heatmap generated (224x224) source=gradcam++ +2026-04-24 23:45:58.926 | INFO | services.ela_service:generate_ela_base64:59 - ELA map generated (2268x4032) +2026-04-24 23:46:00.276 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 2 regions +2026-04-24 23:46:00.291 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=Google, model=Pixel 7 Pro, adjustment=-20 (valid camera metadata (Make/Model/DateTime); GPS coordinates present) +2026-04-24 23:46:00.379 | INFO | api.v1.analyze:analyze_image:230 - Saved AnalysisRecord id=40 score=14 verdict=Very Likely Fake +2026-04-24 23:46:00.382 | ERROR | services.llm_explainer:generate_llm_summary:296 - LLM explainer failed: cannot import name 'genai' from 'google' (unknown location) +2026-04-24 23:46:00.386 | ERROR | services.vlm_breakdown:generate_vlm_breakdown:114 - VLM breakdown failed: cannot import name 'genai' from 'google' (unknown location) +2026-04-24 23:47:37.291 | INFO | services.report_service:generate_report:120 - Report generated id=40 path=temp_reports\deepshield_40_3f0f8ff7.pdf size=14978B +2026-04-24 23:50:59.570 | INFO | api.v1.auth:login:42 - Login user id=6 email=***@gmail.com +2026-04-25 02:48:29.295 | INFO | services.report_service:cleanup_expired:149 - Cleaned up 2 expired reports +2026-04-25 02:48:29.419 | WARNING | services.report_service:cleanup_expired:149 - Cleanup failed for temp_reports\deepshield_40_3f0f8ff7.pdf: [WinError 2] The system cannot find the file specified: 'temp_reports\\deepshield_40_3f0f8ff7.pdf' +2026-04-25 21:48:15.075 | INFO | main:lifespan:108 - Starting DeepShield backend +2026-04-25 21:48:15.082 | INFO | main:lifespan:110 - Database initialized +2026-04-25 21:48:15.082 | INFO | models.model_loader:load_image_model:43 - Loading image model: prithivMLmods/Deep-Fake-Detector-v2-Model +2026-04-25 21:48:18.709 | INFO | models.model_loader:load_image_model:51 - Image model loaded +2026-04-25 21:48:18.712 | INFO | main:lifespan:118 - Shutting down DeepShield backend +2026-04-25 21:52:02.663 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-25 21:52:03.239 | INFO | services.image_service:classify_image:152 - Image classify (average_vit_eff) → Real | vit=0.870 ffpp=n/a eff=0.0529196597635746 → 0.462 +2026-04-25 21:52:04.390 | INFO | models.heatmap_generator:generate_heatmap_base64:186 - Heatmap generated (224x224) source=gradcam++ +2026-04-25 21:52:04.682 | INFO | services.ela_service:generate_ela_base64:59 - ELA map generated (1223x640) +2026-04-25 21:52:05.863 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 5 regions +2026-04-25 21:52:05.883 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=None, model=None, adjustment=0 (no EXIF metadata found) +2026-04-25 21:52:05.927 | INFO | api.v1.analyze:analyze_image:230 - Saved AnalysisRecord id=41 score=46 verdict=Possibly Manipulated +2026-04-25 22:02:22.021 | INFO | main:lifespan:108 - Starting DeepShield backend +2026-04-25 22:02:22.057 | INFO | main:lifespan:110 - Database initialized +2026-04-25 22:02:22.057 | INFO | models.model_loader:load_image_model:43 - Loading image model: prithivMLmods/Deep-Fake-Detector-v2-Model +2026-04-25 22:02:30.014 | INFO | models.model_loader:load_image_model:51 - Image model loaded +2026-04-25 22:13:05.431 | INFO | api.v1.auth:login:42 - Login user id=6 email=***@gmail.com +2026-04-25 22:13:28.224 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-25 22:13:28.471 | INFO | services.image_service:classify_image:152 - Image classify (vit_only) → Fake | vit=0.694 ffpp=n/a eff=n/a → 0.694 +2026-04-25 22:13:28.859 | INFO | models.heatmap_generator:generate_heatmap_base64:176 - EfficientNet heatmap skipped — no face detected +2026-04-25 22:13:31.674 | INFO | services.ela_service:generate_ela_base64:59 - ELA map generated (2268x4032) +2026-04-25 22:13:33.044 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 2 regions +2026-04-25 22:13:33.062 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=Apple, model=iPhone 16 Pro, adjustment=-20 (valid camera metadata (Make/Model/DateTime); GPS coordinates present) +2026-04-25 22:13:33.166 | INFO | api.v1.analyze:analyze_image:230 - Saved AnalysisRecord id=42 score=11 verdict=Very Likely Fake +2026-04-25 22:13:33.169 | ERROR | services.llm_explainer:generate_llm_summary:296 - LLM explainer failed: cannot import name 'genai' from 'google' (unknown location) +2026-04-25 22:13:33.171 | ERROR | services.vlm_breakdown:generate_vlm_breakdown:114 - VLM breakdown failed: cannot import name 'genai' from 'google' (unknown location) +2026-04-26 22:05:50.626 | INFO | main:lifespan:108 - Starting DeepShield backend +2026-04-26 22:05:50.640 | INFO | main:lifespan:110 - Database initialized +2026-04-26 22:05:50.641 | INFO | models.model_loader:load_image_model:44 - Loading image model: prithivMLmods/Deep-Fake-Detector-v2-Model +2026-04-26 22:05:58.170 | INFO | models.model_loader:load_image_model:52 - Image model loaded +2026-04-26 22:07:47.526 | WARNING | models.model_loader:load_ffpp_model:193 - FFPP ViT checkpoint not found at C:\Users\athar\Desktop\trained_models — skipping +2026-04-26 22:07:48.484 | INFO | services.image_service:classify_image:152 - Image classify (average_vit_eff) → Real | vit=0.834 ffpp=n/a eff=0.02755815163254738 → 0.431 +2026-04-26 22:07:50.164 | INFO | models.heatmap_generator:generate_heatmap_base64:186 - Heatmap generated (224x224) source=gradcam++ +2026-04-26 22:07:50.584 | INFO | services.ela_service:generate_ela_base64:59 - ELA map generated (1290x1290) +2026-04-26 22:07:52.661 | INFO | models.heatmap_generator:generate_boxes_base64:232 - Bounding boxes generated: 1 regions +2026-04-26 22:07:52.670 | INFO | services.exif_service:extract_exif:127 - EXIF extracted: make=None, model=None, adjustment=0 (no EXIF metadata found) +2026-04-26 22:07:52.747 | INFO | api.v1.analyze:analyze_image:230 - Saved AnalysisRecord id=43 score=43 verdict=Possibly Manipulated +2026-04-26 22:07:52.752 | ERROR | services.llm_explainer:generate_llm_summary:296 - LLM explainer failed: cannot import name 'genai' from 'google' (unknown location) +2026-04-26 22:07:52.756 | ERROR | services.vlm_breakdown:generate_vlm_breakdown:114 - VLM breakdown failed: cannot import name 'genai' from 'google' (unknown location) +2026-04-26 22:09:45.469 | INFO | services.report_service:generate_report:120 - Report generated id=43 path=temp_reports\deepshield_43_262befa5.pdf size=15602B diff --git a/main.py b/main.py index 2c144c8523c543ab6882943f9f1412ce24d57e75..e1b2c6f44efa181d46577865b32f2090f1ab3e4a 100644 --- a/main.py +++ b/main.py @@ -1,17 +1,98 @@ import asyncio +import secrets +import sys from contextlib import asynccontextmanager from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles from loguru import logger +from slowapi import _rate_limit_exceeded_handler +from slowapi.errors import RateLimitExceeded + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.responses import JSONResponse from api.router import api_router from config import settings from db.database import init_db from models.model_loader import get_model_loader +from services.rate_limit import RateLimitContextMiddleware, limiter from services.report_service import cleanup_expired +class ContentLengthLimitMiddleware(BaseHTTPMiddleware): + """Reject oversized uploads via Content-Length header before reading body. + Saves bandwidth + memory vs letting read_upload_bytes reject post-read.""" + + def __init__(self, app, max_bytes: int) -> None: + super().__init__(app) + self._max = max_bytes + + async def dispatch(self, request, call_next): + cl = request.headers.get("content-length") + if cl and cl.isdigit() and int(cl) > self._max: + return JSONResponse( + status_code=413, + content={"detail": f"Upload exceeds {self._max // (1024 * 1024)} MB limit"}, + ) + return await call_next(request) + + +# === Phase 15.3 — JWT / CORS / logging hardening === + +_DEFAULT_JWT_SECRET = "change-me-in-production" + + +def _enforce_production_hardening() -> None: + """Refuse to start in production with unsafe defaults (Phase 15.3).""" + if settings.JWT_SECRET_KEY == _DEFAULT_JWT_SECRET or not settings.JWT_SECRET_KEY: + example = secrets.token_urlsafe(48) + if settings.DEBUG: + logger.warning( + "JWT_SECRET_KEY is unset or default — safe in dev only. " + f"Set it before deploying. Example: {example}" + ) + else: + logger.error( + "Refusing to start: JWT_SECRET_KEY is unset or default. " + f"Set JWT_SECRET_KEY in your environment. Example: {example}" + ) + sys.exit(1) + if "*" in settings.CORS_ORIGINS and not settings.DEBUG: + logger.error( + "Refusing to start: CORS_ORIGINS contains '*' while allow_credentials=True. " + "Set an explicit origin list." + ) + sys.exit(1) + + +def _configure_logging() -> None: + """Rotate + retain logs, scrub emails.""" + import re + + email_re = re.compile(r"([A-Za-z0-9._%+-]+)@([A-Za-z0-9.-]+\.[A-Za-z]{2,})") + + def _scrub(record): + msg = record["message"] + record["message"] = email_re.sub(r"***@\2", msg) + return True + + logger.remove() + logger.add(sys.stderr, filter=_scrub, level="INFO") + logger.add( + "logs/deepshield.log", + rotation="10 MB", + retention="7 days", + filter=_scrub, + level="INFO", + enqueue=True, + ) + + +_configure_logging() + + async def _report_cleanup_loop(): while True: try: @@ -23,6 +104,7 @@ async def _report_cleanup_loop(): @asynccontextmanager async def lifespan(app: FastAPI): + _enforce_production_hardening() logger.info("Starting DeepShield backend") init_db() logger.info("Database initialized") @@ -43,16 +125,32 @@ app = FastAPI( lifespan=lifespan, ) +# Phase 15.2 — slowapi rate limiter +app.state.limiter = limiter + + +app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) +app.add_middleware(RateLimitContextMiddleware) +# Phase 15.3 — reject oversized uploads before reading body +app.add_middleware(ContentLengthLimitMiddleware, max_bytes=settings.MAX_UPLOAD_SIZE_MB * 1024 * 1024) + +# Phase 15.3 — explicit CORS methods/headers (no wildcards with credentials) app.add_middleware( CORSMiddleware, allow_origins=settings.CORS_ORIGINS, allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], + allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allow_headers=["Authorization", "Content-Type", "Accept", "Origin", "X-Requested-With"], ) app.include_router(api_router) +# Phase 19.2 — serve stored thumbnails / media under /media/* +import os as _os +_media_root = _os.environ.get("MEDIA_ROOT", "./media") +_os.makedirs(_os.path.join(_media_root, "thumbs"), exist_ok=True) +app.mount("/media", StaticFiles(directory=_media_root), name="media") + @app.get("/") def root(): diff --git a/media/03/037d518e19e841c0976352df8d390a7ac9508a4b0d689efd0661ae2db3a92c43.webp b/media/03/037d518e19e841c0976352df8d390a7ac9508a4b0d689efd0661ae2db3a92c43.webp new file mode 100644 index 0000000000000000000000000000000000000000..1e1be9d4a0d1077218c65f72d401d52def215f21 Binary files /dev/null and b/media/03/037d518e19e841c0976352df8d390a7ac9508a4b0d689efd0661ae2db3a92c43.webp differ diff --git a/media/2f/2f7d41a5b57702a9a238409e6a1b973b4398f94c51fdf447e11782ed07693f06.jpg b/media/2f/2f7d41a5b57702a9a238409e6a1b973b4398f94c51fdf447e11782ed07693f06.jpg new file mode 100644 index 0000000000000000000000000000000000000000..74530d3b93a79f035cc2ce8fbcac72a054e8c242 --- /dev/null +++ b/media/2f/2f7d41a5b57702a9a238409e6a1b973b4398f94c51fdf447e11782ed07693f06.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f7d41a5b57702a9a238409e6a1b973b4398f94c51fdf447e11782ed07693f06 +size 2224639 diff --git a/media/50/502e5d7120817956b7ed208987ecad441ef95a527ae8f975340f46669330a27c.jpg b/media/50/502e5d7120817956b7ed208987ecad441ef95a527ae8f975340f46669330a27c.jpg new file mode 100644 index 0000000000000000000000000000000000000000..02177552aa99fcdd502a5d7884987ce24ce0cf8a Binary files /dev/null and b/media/50/502e5d7120817956b7ed208987ecad441ef95a527ae8f975340f46669330a27c.jpg differ diff --git a/media/63/635f21138244fc1dcbff5d0525b3c0a8187b1b9cc0ad90b5bb297a76e7b3850c.jpg b/media/63/635f21138244fc1dcbff5d0525b3c0a8187b1b9cc0ad90b5bb297a76e7b3850c.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0f4b0fbe7e33cf84cff95172bdb01936d5b8eb55 --- /dev/null +++ b/media/63/635f21138244fc1dcbff5d0525b3c0a8187b1b9cc0ad90b5bb297a76e7b3850c.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:635f21138244fc1dcbff5d0525b3c0a8187b1b9cc0ad90b5bb297a76e7b3850c +size 807264 diff --git a/media/6d/6de55b9fc5bdc37898418b7c25d29080f32053a1825e3a7dc2a2ff9df1292015.jpg b/media/6d/6de55b9fc5bdc37898418b7c25d29080f32053a1825e3a7dc2a2ff9df1292015.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3f961d64284d148aaae8453372b0715f25199c75 Binary files /dev/null and b/media/6d/6de55b9fc5bdc37898418b7c25d29080f32053a1825e3a7dc2a2ff9df1292015.jpg differ diff --git a/media/7b/7b626d0ddff59ca602e2e1eb02e62e21093aa647ab53c200ca5203f7fc17f6dd.jpg b/media/7b/7b626d0ddff59ca602e2e1eb02e62e21093aa647ab53c200ca5203f7fc17f6dd.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0a80b4460c2c8628e78547d23d65fbd4da8939b8 --- /dev/null +++ b/media/7b/7b626d0ddff59ca602e2e1eb02e62e21093aa647ab53c200ca5203f7fc17f6dd.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b626d0ddff59ca602e2e1eb02e62e21093aa647ab53c200ca5203f7fc17f6dd +size 4006809 diff --git a/media/bf/bf7ec0c425d20a2161b6a55356a869aad486cf7c6a196420b75be117bf8a47cb.webp b/media/bf/bf7ec0c425d20a2161b6a55356a869aad486cf7c6a196420b75be117bf8a47cb.webp new file mode 100644 index 0000000000000000000000000000000000000000..ab9257ea8ebe3e69887ace616d871963432d0ed8 Binary files /dev/null and b/media/bf/bf7ec0c425d20a2161b6a55356a869aad486cf7c6a196420b75be117bf8a47cb.webp differ diff --git a/media/c0/c064c839c9469d7b616db135f08e09235abd3d73f0889d978d1f92243226a028.jpg b/media/c0/c064c839c9469d7b616db135f08e09235abd3d73f0889d978d1f92243226a028.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ff89295f1fea28869227264262535cc373487e8f --- /dev/null +++ b/media/c0/c064c839c9469d7b616db135f08e09235abd3d73f0889d978d1f92243226a028.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c064c839c9469d7b616db135f08e09235abd3d73f0889d978d1f92243226a028 +size 3127716 diff --git a/media/f0/f0eec5199108c2a4476f9b44aa5454ee0506949b5480b11a6578f2bbcb1f954f.jpg b/media/f0/f0eec5199108c2a4476f9b44aa5454ee0506949b5480b11a6578f2bbcb1f954f.jpg new file mode 100644 index 0000000000000000000000000000000000000000..18823dcae0867cccf629cd6df073a33aa232a215 Binary files /dev/null and b/media/f0/f0eec5199108c2a4476f9b44aa5454ee0506949b5480b11a6578f2bbcb1f954f.jpg differ diff --git a/media/f1/f1c22499ba7787be66a12c32ab2991df97fc4d25c88560207367214e75d7463c.jpg b/media/f1/f1c22499ba7787be66a12c32ab2991df97fc4d25c88560207367214e75d7463c.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cfd12ee0195f1e866f4e9d473249a08f198651d6 Binary files /dev/null and b/media/f1/f1c22499ba7787be66a12c32ab2991df97fc4d25c88560207367214e75d7463c.jpg differ diff --git a/media/thumbs/037d518e19e841c0976352df8d390a7ac9508a4b0d689efd0661ae2db3a92c43_400.jpg b/media/thumbs/037d518e19e841c0976352df8d390a7ac9508a4b0d689efd0661ae2db3a92c43_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..420248eb5e85fcc7eb27ae2e98e1212f646053b7 Binary files /dev/null and b/media/thumbs/037d518e19e841c0976352df8d390a7ac9508a4b0d689efd0661ae2db3a92c43_400.jpg differ diff --git a/media/thumbs/2f7d41a5b57702a9a238409e6a1b973b4398f94c51fdf447e11782ed07693f06_400.jpg b/media/thumbs/2f7d41a5b57702a9a238409e6a1b973b4398f94c51fdf447e11782ed07693f06_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..16a110b4b1bfdda9cac447658a46dfc289bbf5a2 Binary files /dev/null and b/media/thumbs/2f7d41a5b57702a9a238409e6a1b973b4398f94c51fdf447e11782ed07693f06_400.jpg differ diff --git a/media/thumbs/502e5d7120817956b7ed208987ecad441ef95a527ae8f975340f46669330a27c_400.jpg b/media/thumbs/502e5d7120817956b7ed208987ecad441ef95a527ae8f975340f46669330a27c_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9ee86a07e0d2dc03c923eb1a9dbfda838d6c991b Binary files /dev/null and b/media/thumbs/502e5d7120817956b7ed208987ecad441ef95a527ae8f975340f46669330a27c_400.jpg differ diff --git a/media/thumbs/635f21138244fc1dcbff5d0525b3c0a8187b1b9cc0ad90b5bb297a76e7b3850c_400.jpg b/media/thumbs/635f21138244fc1dcbff5d0525b3c0a8187b1b9cc0ad90b5bb297a76e7b3850c_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..465656fc90fb5575d7f4797270016ae40a835fcb Binary files /dev/null and b/media/thumbs/635f21138244fc1dcbff5d0525b3c0a8187b1b9cc0ad90b5bb297a76e7b3850c_400.jpg differ diff --git a/media/thumbs/6de55b9fc5bdc37898418b7c25d29080f32053a1825e3a7dc2a2ff9df1292015_400.jpg b/media/thumbs/6de55b9fc5bdc37898418b7c25d29080f32053a1825e3a7dc2a2ff9df1292015_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..578ade51e577f547369d1b463b197663f40f2c28 Binary files /dev/null and b/media/thumbs/6de55b9fc5bdc37898418b7c25d29080f32053a1825e3a7dc2a2ff9df1292015_400.jpg differ diff --git a/media/thumbs/7b626d0ddff59ca602e2e1eb02e62e21093aa647ab53c200ca5203f7fc17f6dd_400.jpg b/media/thumbs/7b626d0ddff59ca602e2e1eb02e62e21093aa647ab53c200ca5203f7fc17f6dd_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..56141400708de2663036d94e5c54748da496b4ed Binary files /dev/null and b/media/thumbs/7b626d0ddff59ca602e2e1eb02e62e21093aa647ab53c200ca5203f7fc17f6dd_400.jpg differ diff --git a/media/thumbs/bf7ec0c425d20a2161b6a55356a869aad486cf7c6a196420b75be117bf8a47cb_400.jpg b/media/thumbs/bf7ec0c425d20a2161b6a55356a869aad486cf7c6a196420b75be117bf8a47cb_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f23d05fafbce2af307e6d4e4ef9c6e74a1e77d22 Binary files /dev/null and b/media/thumbs/bf7ec0c425d20a2161b6a55356a869aad486cf7c6a196420b75be117bf8a47cb_400.jpg differ diff --git a/media/thumbs/c064c839c9469d7b616db135f08e09235abd3d73f0889d978d1f92243226a028_400.jpg b/media/thumbs/c064c839c9469d7b616db135f08e09235abd3d73f0889d978d1f92243226a028_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..376421962e8737d7c1494c0f772222c8a4982464 Binary files /dev/null and b/media/thumbs/c064c839c9469d7b616db135f08e09235abd3d73f0889d978d1f92243226a028_400.jpg differ diff --git a/media/thumbs/f0eec5199108c2a4476f9b44aa5454ee0506949b5480b11a6578f2bbcb1f954f_400.jpg b/media/thumbs/f0eec5199108c2a4476f9b44aa5454ee0506949b5480b11a6578f2bbcb1f954f_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..360b929b6c365c52b31c78b32284116c27a3c120 Binary files /dev/null and b/media/thumbs/f0eec5199108c2a4476f9b44aa5454ee0506949b5480b11a6578f2bbcb1f954f_400.jpg differ diff --git a/media/thumbs/f1c22499ba7787be66a12c32ab2991df97fc4d25c88560207367214e75d7463c_400.jpg b/media/thumbs/f1c22499ba7787be66a12c32ab2991df97fc4d25c88560207367214e75d7463c_400.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6e96befc2e2c22479d9c88ce3e05bb68f5cb3d19 Binary files /dev/null and b/media/thumbs/f1c22499ba7787be66a12c32ab2991df97fc4d25c88560207367214e75d7463c_400.jpg differ diff --git a/model_loader.py b/model_loader.py deleted file mode 100644 index d71e9f3f59bb7a51d81cbd8d82ef940d521118e0..0000000000000000000000000000000000000000 --- a/model_loader.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -from threading import Lock -from typing import Optional, Tuple - -from loguru import logger - -from config import settings - - -class ModelLoader: - """Singleton holder for preloaded AI models. Thread-safe lazy init.""" - - _instance: Optional["ModelLoader"] = None - _lock: Lock = Lock() - - def __new__(cls) -> "ModelLoader": - if cls._instance is None: - with cls._lock: - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance._image_model = None - cls._instance._image_processor = None - cls._instance._text_pipeline = None - cls._instance._multilang_text_pipeline = None - cls._instance._ocr_reader = None - cls._instance._face_detector = None - cls._instance._spacy_nlp = None - cls._instance._sentence_transformer = None - return cls._instance - - @classmethod - def get_instance(cls) -> "ModelLoader": - return cls() - - # ---------- Image (ViT deepfake classifier) ---------- - def load_image_model(self) -> Tuple[object, object]: - if self._image_model is None: - logger.info(f"Loading image model: {settings.IMAGE_MODEL_ID}") - from transformers import AutoImageProcessor, AutoModelForImageClassification - - self._image_processor = AutoImageProcessor.from_pretrained(settings.IMAGE_MODEL_ID) - model = AutoModelForImageClassification.from_pretrained(settings.IMAGE_MODEL_ID) - model.to(settings.DEVICE) - model.eval() - self._image_model = model - logger.info("Image model loaded") - return self._image_model, self._image_processor - - # ---------- Text (BERT fake-news classifier — English) ---------- - def load_text_model(self): - if self._text_pipeline is None: - logger.info(f"Loading text model: {settings.TEXT_MODEL_ID}") - from transformers import pipeline - - self._text_pipeline = pipeline( - "text-classification", - model=settings.TEXT_MODEL_ID, - device=0 if settings.DEVICE == "cuda" else -1, - ) - logger.info("Text model loaded") - return self._text_pipeline - - # ---------- Multilingual text model (Phase 13) ---------- - def load_multilang_text_model(self): - """Load multilingual fake-news classifier. Falls back to English model if not configured.""" - model_id = settings.TEXT_MULTILANG_MODEL_ID - if not model_id: - logger.debug("TEXT_MULTILANG_MODEL_ID not set — falling back to English text model") - return self.load_text_model() - - if self._multilang_text_pipeline is None: - logger.info(f"Loading multilingual text model: {model_id}") - from transformers import pipeline - - self._multilang_text_pipeline = pipeline( - "text-classification", - model=model_id, - device=0 if settings.DEVICE == "cuda" else -1, - ) - logger.info("Multilingual text model loaded") - return self._multilang_text_pipeline - - # ---------- spaCy NLP (Phase 13 NER) ---------- - def load_spacy_nlp(self): - """Lazy-load spaCy English NLP model. Returns None if spaCy is not installed.""" - if self._spacy_nlp is None: - try: - import spacy # type: ignore - try: - self._spacy_nlp = spacy.load("en_core_web_sm") - logger.info("spaCy en_core_web_sm loaded") - except OSError: - logger.warning( - "spaCy model 'en_core_web_sm' not found. " - "Run: python -m spacy download en_core_web_sm" - ) - return None - except ImportError: - logger.warning("spaCy not installed — NER keyword extraction disabled") - return None - return self._spacy_nlp - - # ---------- Sentence-Transformer (Phase 13 truth-override) ---------- - def load_sentence_transformer(self): - """Lazy-load sentence-transformers/all-MiniLM-L6-v2. Returns None if not installed.""" - if self._sentence_transformer is None: - try: - from sentence_transformers import SentenceTransformer # type: ignore - self._sentence_transformer = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") - logger.info("Sentence-transformer (all-MiniLM-L6-v2) loaded") - except ImportError: - logger.warning("sentence-transformers not installed — truth-override disabled") - return None - except Exception as e: - logger.warning(f"Sentence-transformer load failed: {e}") - return None - return self._sentence_transformer - - # ---------- OCR (EasyOCR) — Phase 13: use OCR_LANGS from config ---------- - def load_ocr_engine(self): - if self._ocr_reader is None: - langs = [l.strip() for l in settings.OCR_LANGS.split(",") if l.strip()] - if not langs: - langs = ["en"] - logger.info(f"Loading EasyOCR reader (langs: {langs})") - import easyocr # type: ignore - - self._ocr_reader = easyocr.Reader( - langs, gpu=(settings.DEVICE == "cuda"), verbose=False, download_enabled=True, - ) - logger.info("EasyOCR loaded") - return self._ocr_reader - - # ---------- Face detector (MediaPipe) ---------- - def load_face_detector(self): - if self._face_detector is None: - logger.info("Loading MediaPipe FaceMesh") - import mediapipe as mp # type: ignore - - self._face_detector = mp.solutions.face_mesh.FaceMesh( - static_image_mode=True, - max_num_faces=5, - min_detection_confidence=0.5, - ) - logger.info("MediaPipe FaceMesh loaded") - return self._face_detector - - # ---------- Preload ---------- - def preload_phase1(self) -> None: - """Preload only what Phase 1 needs (image model).""" - self.load_image_model() - - -def get_model_loader() -> ModelLoader: - return ModelLoader.get_instance() diff --git a/models.py b/models.py deleted file mode 100644 index af3b2f8f14b6485f08ed933ec490c11d10802e4a..0000000000000000000000000000000000000000 --- a/models.py +++ /dev/null @@ -1,45 +0,0 @@ -from datetime import datetime - -from sqlalchemy import DateTime, ForeignKey, Integer, String, Text -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from db.database import Base - - -class User(Base): - __tablename__ = "users" - - id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) - email: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False) - password_hash: Mapped[str] = mapped_column(String(255), nullable=False) - name: Mapped[str | None] = mapped_column(String(255), nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) - - analyses: Mapped[list["AnalysisRecord"]] = relationship(back_populates="user") - - -class AnalysisRecord(Base): - __tablename__ = "analyses" - - id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) - user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) - media_type: Mapped[str] = mapped_column(String(32), nullable=False) # image|video|text|screenshot - verdict: Mapped[str] = mapped_column(String(32), nullable=False) - authenticity_score: Mapped[float] = mapped_column(nullable=False) - result_json: Mapped[str] = mapped_column(Text, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) - - user: Mapped["User | None"] = relationship(back_populates="analyses") - report: Mapped["Report | None"] = relationship(back_populates="analysis", uselist=False) - - -class Report(Base): - __tablename__ = "reports" - - id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) - analysis_id: Mapped[int] = mapped_column(ForeignKey("analyses.id"), nullable=False) - file_path: Mapped[str] = mapped_column(String(512), nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) - expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) - - analysis: Mapped["AnalysisRecord"] = relationship(back_populates="report") diff --git a/models/heatmap_generator.py b/models/heatmap_generator.py index b43dc7cf71adc80a981bc401ac7ffbb8e15e9b21..03e05cf1c4fd714c3e751c58c062a20dec00e733 100644 --- a/models/heatmap_generator.py +++ b/models/heatmap_generator.py @@ -107,14 +107,29 @@ def _compute_gradcam_pp( return grayscale_cam, rgb_float +def _face_bbox_from_detections(frame_data: dict, orig_h: int, orig_w: int) -> Optional[tuple[int,int,int,int]]: + """Extract (ymin, xmin, ymax, xmax) in pixel coords from BlazeFace frame_data.""" + detections = frame_data.get("detections", []) + if len(detections) == 0: + return None + d = detections[0] # first (highest-confidence) face + ymin = int(max(0, d[0])) + xmin = int(max(0, d[1])) + ymax = int(min(orig_h, d[2])) + xmax = int(min(orig_w, d[3])) + if ymax <= ymin or xmax <= xmin: + return None + return ymin, xmin, ymax, xmax + + def _compute_gradcam_pp_efficientnet( pil_img: Image.Image, -) -> tuple[np.ndarray, np.ndarray, Literal["attention", "gradcam++"]]: +) -> tuple[np.ndarray, Optional[tuple[int,int,int,int]], Literal["attention", "gradcam++"]]: """Grad-CAM++ for EfficientNetAutoAttB4. - Returns (grayscale_cam, rgb_float, heatmap_source). - Prefers the model's built-in attention map; falls back to Grad-CAM++ on the - last MBConv block if attention extraction fails. + Returns (grayscale_cam_224, face_bbox_pixels_or_None, heatmap_source). + grayscale_cam_224 is in the 224x224 coordinate space of the face crop. + face_bbox_pixels is (ymin, xmin, ymax, xmax) in original image pixels. """ loader = get_model_loader() eff = loader.load_efficientnet() @@ -124,39 +139,58 @@ def _compute_gradcam_pp_efficientnet( if pil_img.mode != "RGB": pil_img = pil_img.convert("RGB") img_np = np.array(pil_img) + orig_h, orig_w = img_np.shape[:2] - # Prepare face crop (same path as detect_image). frame_data = eff.face_extractor.process_image(img=img_np) faces: list = frame_data.get("faces", []) if not faces: raise ValueError("no_face") - face_t = eff._face_tensor(faces[0]).unsqueeze(0).to(eff.device) + face_bbox = _face_bbox_from_detections(frame_data, orig_h, orig_w) - # Resize the face crop to float [0,1] for overlay. - face_np = faces[0] - h, w = face_np.shape[:2] - rgb_float = face_np.astype(np.float32) / 255.0 - if rgb_float.shape[:2] != (224, 224): - rgb_float = cv2.resize(rgb_float, (224, 224)).astype(np.float32) + face_t = eff._face_tensor(faces[0]).unsqueeze(0).to(eff.device) - # Try Grad-CAM++ on last MBConv block (_blocks[-1]). try: net = eff.net target_layers = [net.efficientnet._blocks[-1]] - face_t.requires_grad_(True) for p in net.parameters(): p.requires_grad_(True) - with GradCAMPlusPlus(model=net, target_layers=target_layers) as cam: grayscale_cam = cam(input_tensor=face_t, targets=None)[0] - - return grayscale_cam, rgb_float, "gradcam++" + return grayscale_cam, face_bbox, "gradcam++" except Exception as e: logger.warning(f"EfficientNet Grad-CAM++ failed ({e}), using uniform fallback") grayscale_cam = np.ones((224, 224), dtype=np.float32) * 0.5 - return grayscale_cam, rgb_float, "gradcam++" + return grayscale_cam, face_bbox, "gradcam++" + + +def _cam_to_full_image( + grayscale_cam: np.ndarray, + pil_img: Image.Image, + face_bbox: Optional[tuple[int,int,int,int]] = None, +) -> tuple[np.ndarray, np.ndarray]: + """Resize grayscale_cam to the original image dimensions. + + For EfficientNet (face-crop cam + known bbox): places the cam activation + at the face location; background activation is 0. + For ViT (full-image cam): bilinear resize to original dims. + + Returns (cam_full [H,W] float32), orig_np [H,W,3] float32 in [0,1]). + """ + orig_w, orig_h = pil_img.size + orig_np = np.array(pil_img.convert("RGB")).astype(np.float32) / 255.0 + + if face_bbox is not None: + ymin, xmin, ymax, xmax = face_bbox + face_h, face_w = ymax - ymin, xmax - xmin + cam_full = np.zeros((orig_h, orig_w), dtype=np.float32) + cam_resized = cv2.resize(grayscale_cam, (face_w, face_h), interpolation=cv2.INTER_LINEAR) + cam_full[ymin:ymax, xmin:xmax] = cam_resized + else: + cam_full = cv2.resize(grayscale_cam, (orig_w, orig_h), interpolation=cv2.INTER_LINEAR) + + return cam_full, orig_np def generate_heatmap_base64( @@ -164,26 +198,34 @@ def generate_heatmap_base64( target_class_idx: Optional[int] = None, model_family: Literal["vit", "efficientnet"] = "vit", ) -> tuple[str, str]: - """Produce a base64 data-URL PNG of the Grad-CAM++ overlay. + """Produce a base64 data-URL PNG of the Grad-CAM++ overlay at original image resolution. - Returns (base64_png, heatmap_source) where heatmap_source is one of - "gradcam++", "attention", "fallback", "none". + Returns (base64_png, heatmap_source). """ if model_family == "efficientnet": try: - grayscale_cam, rgb_float, source = _compute_gradcam_pp_efficientnet(pil_img) + grayscale_cam, face_bbox, source = _compute_gradcam_pp_efficientnet(pil_img) + cam_full, orig_np = _cam_to_full_image(grayscale_cam, pil_img, face_bbox) except ValueError: - logger.info("EfficientNet heatmap skipped — no face detected") - return "", "none" + # BlazeFace found no face — fall back to ViT Grad-CAM on the full image. + logger.info("EfficientNet heatmap: no face detected — falling back to ViT Grad-CAM++") + try: + grayscale_cam, _ = _compute_gradcam_pp(pil_img, target_class_idx) + cam_full, orig_np = _cam_to_full_image(grayscale_cam, pil_img, None) + source = "vit_fallback" + except Exception as fe: + logger.warning(f"ViT fallback heatmap also failed: {fe}") + return "", "none" except Exception as e: logger.warning(f"EfficientNet heatmap failed: {e}") return "", "fallback" else: - grayscale_cam, rgb_float = _compute_gradcam_pp(pil_img, target_class_idx) + grayscale_cam, _ = _compute_gradcam_pp(pil_img, target_class_idx) source = "gradcam++" + cam_full, orig_np = _cam_to_full_image(grayscale_cam, pil_img, None) - overlay = show_cam_on_image(rgb_float, grayscale_cam, use_rgb=True) - logger.info(f"Heatmap generated ({overlay.shape[0]}x{overlay.shape[1]}) source={source}") + overlay = show_cam_on_image(orig_np, cam_full, use_rgb=True) + logger.info(f"Heatmap generated ({overlay.shape[1]}x{overlay.shape[0]}) source={source}") return _encode_overlay_to_base64(overlay), source @@ -193,41 +235,46 @@ def generate_boxes_base64( top_k: int = 5, threshold: float = 0.4, ) -> str: - """Produce bounding boxes around top-K connected components from Grad-CAM++ activation. - Renders colored boxes (red/yellow/orange by intensity) on the original image. + """Draw Grad-CAM++ activation bounding boxes on the full original image. + + Uses the ViT cam (full-image coverage), resizes it to original dimensions, + finds contours, and draws boxes at the correct pixel locations. """ - grayscale_cam, rgb_float = _compute_gradcam_pp(pil_img, target_class_idx) + grayscale_cam, _ = _compute_gradcam_pp(pil_img, target_class_idx) - h, w = rgb_float.shape[:2] - base_img = (rgb_float * 255).astype(np.uint8).copy() + # Use original image as the canvas — resize cam to match + orig_w, orig_h = pil_img.size + base_img = np.array(pil_img.convert("RGB")).copy() + cam_full = cv2.resize(grayscale_cam, (orig_w, orig_h), interpolation=cv2.INTER_LINEAR) - # Threshold the heatmap to find activated regions - binary = (grayscale_cam >= threshold).astype(np.uint8) * 255 + binary = (cam_full >= threshold).astype(np.uint8) * 255 contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: logger.info("No significant activation regions found for bounding boxes") return _encode_overlay_to_base64(base_img) - # Sort by area descending, take top_k contours = sorted(contours, key=cv2.contourArea, reverse=True)[:top_k] - # Color by mean activation intensity within each box + # Scale line width to image size + line_w = max(2, orig_w // 300) + font_scale = max(0.5, orig_w / 1200) + for cnt in contours: x, y, bw, bh = cv2.boundingRect(cnt) - region_activation = grayscale_cam[y:y + bh, x:x + bw].mean() + region_activation = cam_full[y:y + bh, x:x + bw].mean() if region_activation >= 0.7: - color = (220, 40, 40) # red — high suspicion + color = (220, 40, 40) elif region_activation >= 0.5: - color = (240, 140, 20) # orange — medium + color = (240, 140, 20) else: - color = (230, 200, 40) # yellow — lower + color = (230, 200, 40) - cv2.rectangle(base_img, (x, y), (x + bw, y + bh), color, 2) + cv2.rectangle(base_img, (x, y), (x + bw, y + bh), color, line_w) label = f"{region_activation * 100:.0f}%" - cv2.putText(base_img, label, (x, max(y - 6, 12)), - cv2.FONT_HERSHEY_SIMPLEX, 0.4, color, 1, cv2.LINE_AA) + cv2.putText(base_img, label, (x, max(y - 6, 14)), + cv2.FONT_HERSHEY_SIMPLEX, font_scale, color, line_w, cv2.LINE_AA) - logger.info(f"Bounding boxes generated: {len(contours)} regions") + logger.info(f"Bounding boxes generated: {len(contours)} regions on {orig_w}x{orig_h} image") return _encode_overlay_to_base64(base_img) diff --git a/models/icpr2020dfdc/.gitignore b/models/icpr2020dfdc/.gitignore deleted file mode 100644 index 10ffe55501d4fd4520137e6af1a7e65ba6a9b7e0..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ - -.idea/ -.DS_Store -.ipynb_checkpoints/ -__pycache__/ \ No newline at end of file diff --git a/models/icpr2020dfdc/.travis.yml b/models/icpr2020dfdc/.travis.yml deleted file mode 100644 index e6d3c37919f6a468d61b66ec2b8bdfeb63634365..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: python -python: - - "3.6.9" -install: - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O $HOME/miniconda.sh - - bash $HOME/miniconda.sh -bfp $HOME/miniconda3 - - export PATH=$HOME/miniconda3/bin:$PATH - - conda env create -f environment.yml -before_script: - - source activate icpr2020 - - cd test -script: - - python -m unittest test_dfdc.TestDFDC - - python -m unittest test_ffpp.TestFFPP - diff --git a/models/icpr2020dfdc/LICENSE b/models/icpr2020dfdc/LICENSE deleted file mode 100644 index f288702d2fa16d3cdf0035b15a9fcbc552cd88e7..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/LICENSE +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - <program> Copyright (C) <year> <name of author> - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -<https://www.gnu.org/licenses/>. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -<https://www.gnu.org/licenses/why-not-lgpl.html>. diff --git a/models/icpr2020dfdc/README.md b/models/icpr2020dfdc/README.md deleted file mode 100644 index 88c15f3c5d080f18772e59965d485e0ad6cf2f6d..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/README.md +++ /dev/null @@ -1,120 +0,0 @@ -# Video Face Manipulation Detection Through Ensemble of CNNs -[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/video-face-manipulation-detection-through/deepfake-detection-on-dfdc)](https://paperswithcode.com/sota/deepfake-detection-on-dfdc?p=video-face-manipulation-detection-through) -[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/video-face-manipulation-detection-through/deepfake-detection-on-faceforensics-1)](https://paperswithcode.com/sota/deepfake-detection-on-faceforensics-1?p=video-face-manipulation-detection-through) -[![Build Status](https://travis-ci.org/polimi-ispl/icpr2020dfdc.svg?branch=master)](https://travis-ci.org/polimi-ispl/icpr2020dfdc) - -![](assets/faces_attention.png) - -<p align='center'> - <img src='assets/mqzvfufzoq_face.gif'/> - <img src='assets/mqzvfufzoq_face_att.gif'/> -</p> - -This is the official repository of **Video Face Manipulation Detection Through Ensemble of CNNs**, -presented at [ICPR2020](https://www.micc.unifi.it/icpr2020/) and currently available on [IEEExplore](https://ieeexplore.ieee.org/document/9412711) and [arXiv](https://arxiv.org/abs/2004.07676). -If you use this repository for your research, please consider citing our paper. Refer to [How to cite](https://github.com/polimi-ispl/icpr2020dfdc#how-to-cite) section to get the correct entry for your bibliography. - -We participated as the **ISPL** team in the [Kaggle Deepfake Detection Challenge](https://www.kaggle.com/c/deepfake-detection-challenge/). -With this implementation, we reached the 41st position over 2116 teams (**top 2%**) on the [private leaderboard](https://www.kaggle.com/c/deepfake-detection-challenge/leaderboard). - -This repository is currently under maintenance, if you are experiencing any problems, please open an [issue](https://github.com/polimi-ispl/icpr2020dfdc/issues). -## Getting started - -### Prerequisites -- Install [conda](https://docs.conda.io/en/latest/miniconda.html) -- Create the `icpr2020` environment with *environment.yml* -```bash -$ conda env create -f environment.yml -$ conda activate icpr2020 -``` -- Download and unzip the [datasets](#datasets) - -### Quick run -If you just want to test the pre-trained models against your own videos or images: -- [Video prediction notebook](https://github.com/polimi-ispl/icpr2020dfdc/blob/master/notebook/Video%20prediction.ipynb) <a target="_blank" href="https://colab.research.google.com/drive/12WnvmerHBNbJ49HdoH1lli_O8SwaFPjv?usp=sharing"> - <img src="https://colab.research.google.com/assets/colab-badge.svg"> -</a> - -- [Image prediction notebook](https://github.com/polimi-ispl/icpr2020dfdc/blob/master/notebook/Image%20prediction.ipynb) <a target="_blank" href="https://colab.research.google.com/drive/19oVKlzEr58VZfRnSq-nW8kFYuxkh3GM8?usp=sharing"> - <img src="https://colab.research.google.com/assets/colab-badge.svg"> -</a> - -- [Image prediction with attention](notebook/Image%20prediction%20and%20attention.ipynb) <a target="_blank" href="https://colab.research.google.com/drive/1zcglis2Qx2vtJhrogn8aKA-mbUotLZLK?usp=sharing"> - <img src="https://colab.research.google.com/assets/colab-badge.svg"> -</a> - -### The whole pipeline -You need to preprocess the datasets in order to index all the samples and extract faces. Just run the script [make_dataset.sh](scripts/make_dataset.sh) - -```bash -$ ./scripts/make_dataset.sh -``` - -Please note that we use only 32 frames per video. You can easily tweak this parameter in [extract_faces.py](extract_faces.py) -Also, please note that **for the DFDC** we have resorted to _the training split_ exclusively! -In `scripts/make_dataset.sh` the value of `DFDC_SRC` should point to the directory containing the DFDC train split. - - -### Celeb-DF (v2) -Altough **we did not use this dataset in the paper**, we provide a script [index_celebdf.py](index_celebdf.py) to index the videos similarly to -DFDC and FF++. Once you have the index, you can proceed with the pipeline starting from [extract_faces.py](extract_faces.py). You can also use the -split `celebdf` during training/testing. - -### Train -In [train_all.sh](scripts/train_all.sh) you can find a comprehensive list of all the commands to train the models presented in the paper. -Please refer to the comments in the script for hints on their usage. - -#### Training a single model -If you want to train some models without lunching the script: -- for the **non-siamese** architectures (e.g. EfficientNetB4, EfficientNetB4Att), you can simply specify the model in [train_binclass.py](train_binclass.py) with the *--net* parameter; -- for the **siamese** architectures (e.g. EfficientNetB4ST, EfficientNetB4AttST), you have to: - 1. train the architecture as a feature extractor first, using the [train_triplet.py](train_triplet.py) script and being careful of specifying its name with the *--net* parameter **without** the ST suffix. For instance, for training the EfficientNetB4ST you will have to first run `python train_triplet.py --net EfficientNetB4 --otherparams`; - 2. finetune the model using [train_binclass.py](train_binclass.py), being careful this time to specify the architecture's name **with** the ST suffix and to insert as *--init* argument the path to the weights of the feature extractor trained at the previous step. You will end up running something like `python train_binclass.py --net EfficientNetB4ST --init path/to/EfficientNetB4/weights/trained/with/train_triplet/weights.pth --otherparams` - -### Test -In [test_all.sh](scripts/test_all.sh) you can find a comprehensive list of all the commands for testing the models presented in the paper. - -#### Pretrained weights -We also provide pretrained weights for all the architectures presented in the paper. -Please refer to this [Dropbox link](https://www.dropbox.com/sh/cesamx5ytd5j08c/AADG_eEmhskliMaT0Gbk-yHDa?dl=0). -Each directory is named `$NETWORK_$DATASET` where `$NETWORK` is the architecture name and `$DATASET` is the training dataset. -In each directory, you can find `bestval.pth` which are the best network weights according to the validation set. - - -Additionally, you can find Jupyter notebooks for results computations in the [notebook](notebook) folder. - - -## Datasets -- [Facebook's DeepFake Detection Challenge (DFDC) train dataset](https://www.kaggle.com/c/deepfake-detection-challenge/data) | [arXiv paper](https://arxiv.org/abs/2006.07397) -- [FaceForensics++](https://github.com/ondyari/FaceForensics/blob/master/dataset/README.md) | [arXiv paper](https://arxiv.org/abs/1901.08971) -- [Celeb-DF (v2)](http://www.cs.albany.edu/~lsw/celeb-deepfakeforensics.html) | [arXiv paper](https://arxiv.org/abs/1909.12962) (**Just for reference, not used in the paper**) - -## References -- [EfficientNet PyTorch](https://github.com/lukemelas/EfficientNet-PyTorch) -- [Xception PyTorch](https://github.com/tstandley/Xception-PyTorch) - -## How to cite -Plain text: -``` -N. Bonettini, E. D. Cannas, S. Mandelli, L. Bondi, P. Bestagini and S. Tubaro, "Video Face Manipulation Detection Through Ensemble of CNNs," 2020 25th International Conference on Pattern Recognition (ICPR), 2021, pp. 5012-5019, doi: 10.1109/ICPR48806.2021.9412711. -``` - -Bibtex: -```bibtex -@INPROCEEDINGS{9412711, - author={Bonettini, Nicolò and Cannas, Edoardo Daniele and Mandelli, Sara and Bondi, Luca and Bestagini, Paolo and Tubaro, Stefano}, - booktitle={2020 25th International Conference on Pattern Recognition (ICPR)}, - title={Video Face Manipulation Detection Through Ensemble of CNNs}, - year={2021}, - volume={}, - number={}, - pages={5012-5019}, - doi={10.1109/ICPR48806.2021.9412711}} -``` -## Credits -[Image and Sound Processing Lab - Politecnico di Milano](http://ispl.deib.polimi.it/) -- Nicolò Bonettini -- Edoardo Daniele Cannas -- Sara Mandelli -- Luca Bondi -- Paolo Bestagini diff --git a/models/icpr2020dfdc/architectures/__init__.py b/models/icpr2020dfdc/architectures/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/models/icpr2020dfdc/architectures/externals/__init__.py b/models/icpr2020dfdc/architectures/externals/__init__.py deleted file mode 100644 index b0f7fac88c5037fcb193abd72972bb966640b691..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/architectures/externals/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .xception import xception diff --git a/models/icpr2020dfdc/architectures/externals/xception.py b/models/icpr2020dfdc/architectures/externals/xception.py deleted file mode 100644 index c250bc93a4ec48587973db52d8319cd0c423edf5..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/architectures/externals/xception.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -Ported to pytorch thanks to [tstandley](https://github.com/tstandley/Xception-PyTorch) - -@author: tstandley -Adapted by cadene - -Creates an Xception Model as defined in: - -Francois Chollet -Xception: Deep Learning with Depthwise Separable Convolutions -https://arxiv.org/pdf/1610.02357.pdf - -This weights ported from the Keras implementation. Achieves the following performance on the validation set: - -Loss:0.9173 Prec@1:78.892 Prec@5:94.292 - -REMEMBER to set your image size to 3x299x299 for both test and validation - -normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5], - std=[0.5, 0.5, 0.5]) - -The resize parameter of the validation transform should be 333, and make sure to center crop at 299x299 -""" -from __future__ import print_function, division, absolute_import - -import torch.nn as nn -import torch.nn.functional as F -import torch.utils.model_zoo as model_zoo - -__all__ = ['xception'] - -pretrained_settings = { - 'xception': { - 'imagenet': { - 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/xception-43020ad28.pth', - 'input_space': 'RGB', - 'input_size': [3, 299, 299], - 'input_range': [0, 1], - 'mean': [0.5, 0.5, 0.5], - 'std': [0.5, 0.5, 0.5], - 'num_classes': 1000, - 'scale': 0.8975 - # The resize parameter of the validation transform should be 333, and make sure to center crop at 299x299 - } - } -} - - -class SeparableConv2d(nn.Module): - def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, padding=0, dilation=1, bias=False): - super(SeparableConv2d, self).__init__() - - self.conv1 = nn.Conv2d(in_channels, in_channels, kernel_size, stride, padding, dilation, groups=in_channels, - bias=bias) - self.pointwise = nn.Conv2d(in_channels, out_channels, 1, 1, 0, 1, 1, bias=bias) - - def forward(self, x): - x = self.conv1(x) - x = self.pointwise(x) - return x - - -class Block(nn.Module): - def __init__(self, in_filters, out_filters, reps, strides=1, start_with_relu=True, grow_first=True): - super(Block, self).__init__() - - if out_filters != in_filters or strides != 1: - self.skip = nn.Conv2d(in_filters, out_filters, 1, stride=strides, bias=False) - self.skipbn = nn.BatchNorm2d(out_filters) - else: - self.skip = None - - rep = [] - - filters = in_filters - if grow_first: - rep.append(nn.ReLU(inplace=True)) - rep.append(SeparableConv2d(in_filters, out_filters, 3, stride=1, padding=1, bias=False)) - rep.append(nn.BatchNorm2d(out_filters)) - filters = out_filters - - for i in range(reps - 1): - rep.append(nn.ReLU(inplace=True)) - rep.append(SeparableConv2d(filters, filters, 3, stride=1, padding=1, bias=False)) - rep.append(nn.BatchNorm2d(filters)) - - if not grow_first: - rep.append(nn.ReLU(inplace=True)) - rep.append(SeparableConv2d(in_filters, out_filters, 3, stride=1, padding=1, bias=False)) - rep.append(nn.BatchNorm2d(out_filters)) - - if not start_with_relu: - rep = rep[1:] - else: - rep[0] = nn.ReLU(inplace=False) - - if strides != 1: - rep.append(nn.MaxPool2d(3, strides, 1)) - self.rep = nn.Sequential(*rep) - - def forward(self, inp): - x = self.rep(inp) - - if self.skip is not None: - skip = self.skip(inp) - skip = self.skipbn(skip) - else: - skip = inp - - x += skip - return x - - -class Xception(nn.Module): - """ - Xception optimized for the ImageNet dataset, as specified in - https://arxiv.org/pdf/1610.02357.pdf - """ - - def __init__(self, num_classes=1000): - """ Constructor - Args: - num_classes: number of classes - """ - super(Xception, self).__init__() - self.num_classes = num_classes - - self.conv1 = nn.Conv2d(3, 32, 3, 2, 0, bias=False) - self.bn1 = nn.BatchNorm2d(32) - self.relu1 = nn.ReLU(inplace=True) - - self.conv2 = nn.Conv2d(32, 64, 3, bias=False) - self.bn2 = nn.BatchNorm2d(64) - self.relu2 = nn.ReLU(inplace=True) - # do relu here - - self.block1 = Block(64, 128, 2, 2, start_with_relu=False, grow_first=True) - self.block2 = Block(128, 256, 2, 2, start_with_relu=True, grow_first=True) - self.block3 = Block(256, 728, 2, 2, start_with_relu=True, grow_first=True) - - self.block4 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) - self.block5 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) - self.block6 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) - self.block7 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) - - self.block8 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) - self.block9 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) - self.block10 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) - self.block11 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) - - self.block12 = Block(728, 1024, 2, 2, start_with_relu=True, grow_first=False) - - self.conv3 = SeparableConv2d(1024, 1536, 3, 1, 1) - self.bn3 = nn.BatchNorm2d(1536) - self.relu3 = nn.ReLU(inplace=True) - - # do relu here - self.conv4 = SeparableConv2d(1536, 2048, 3, 1, 1) - self.bn4 = nn.BatchNorm2d(2048) - - self.fc = nn.Linear(2048, num_classes) - - # #------- init weights -------- - # for m in self.modules(): - # if isinstance(m, nn.Conv2d): - # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - # m.weight.data.normal_(0, math.sqrt(2. / n)) - # elif isinstance(m, nn.BatchNorm2d): - # m.weight.data.fill_(1) - # m.bias.data.zero_() - # #----------------------------- - - def features(self, input): - x = self.conv1(input) - x = self.bn1(x) - x = self.relu1(x) - - x = self.conv2(x) - x = self.bn2(x) - x = self.relu2(x) - - x = self.block1(x) - x = self.block2(x) - x = self.block3(x) - x = self.block4(x) - x = self.block5(x) - x = self.block6(x) - x = self.block7(x) - x = self.block8(x) - x = self.block9(x) - x = self.block10(x) - x = self.block11(x) - x = self.block12(x) - - x = self.conv3(x) - x = self.bn3(x) - x = self.relu3(x) - - x = self.conv4(x) - x = self.bn4(x) - return x - - def logits(self, features): - x = nn.ReLU(inplace=True)(features) - - x = F.adaptive_avg_pool2d(x, (1, 1)) - x = x.view(x.size(0), -1) - x = self.last_linear(x) - return x - - def forward(self, input): - x = self.features(input) - x = self.logits(x) - return x - - -def xception(num_classes=1000, pretrained='imagenet'): - model = Xception(num_classes=num_classes) - if pretrained: - settings = pretrained_settings['xception'][pretrained] - assert num_classes == settings['num_classes'], \ - "num_classes should be {}, but is {}".format(settings['num_classes'], num_classes) - - model = Xception(num_classes=num_classes) - model.load_state_dict(model_zoo.load_url(settings['url'])) - - model.input_space = settings['input_space'] - model.input_size = settings['input_size'] - model.input_range = settings['input_range'] - model.mean = settings['mean'] - model.std = settings['std'] - - # TODO: ugly - model.last_linear = model.fc - del model.fc - return model diff --git a/models/icpr2020dfdc/architectures/fornet.py b/models/icpr2020dfdc/architectures/fornet.py deleted file mode 100644 index b227c120f241bbf5281e19ff186df2adde4efa25..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/architectures/fornet.py +++ /dev/null @@ -1,245 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -from collections import OrderedDict - -import torch -from efficientnet_pytorch import EfficientNet -from torch import nn as nn -from torch.nn import functional as F -from torchvision import transforms - -from . import externals - -""" -Feature Extractor -""" - - -class FeatureExtractor(nn.Module): - """ - Abstract class to be extended when supporting features extraction. - It also provides standard normalized and parameters - """ - - def features(self, x: torch.Tensor) -> torch.Tensor: - raise NotImplementedError - - def get_trainable_parameters(self): - return self.parameters() - - @staticmethod - def get_normalizer(): - return transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - - -""" -EfficientNet -""" - - -class EfficientNetGen(FeatureExtractor): - def __init__(self, model: str): - super(EfficientNetGen, self).__init__() - - self.efficientnet = EfficientNet.from_pretrained(model) - self.classifier = nn.Linear(self.efficientnet._conv_head.out_channels, 1) - del self.efficientnet._fc - - def features(self, x: torch.Tensor) -> torch.Tensor: - x = self.efficientnet.extract_features(x) - x = self.efficientnet._avg_pooling(x) - x = x.flatten(start_dim=1) - return x - - def forward(self, x): - x = self.features(x) - x = self.efficientnet._dropout(x) - x = self.classifier(x) - return x - - -class EfficientNetB4(EfficientNetGen): - def __init__(self): - super(EfficientNetB4, self).__init__(model='efficientnet-b4') - - -""" -EfficientNetAutoAtt -""" - - -class EfficientNetAutoAtt(EfficientNet): - def init_att(self, model: str, width: int): - """ - Initialize attention - :param model: efficientnet-bx, x \in {0,..,7} - :param depth: attention width - :return: - """ - if model == 'efficientnet-b4': - self.att_block_idx = 9 - if width == 0: - self.attconv = nn.Conv2d(kernel_size=1, in_channels=56, out_channels=1) - else: - attconv_layers = [] - for i in range(width): - attconv_layers.append( - ('conv{:d}'.format(i), nn.Conv2d(kernel_size=3, padding=1, in_channels=56, out_channels=56))) - attconv_layers.append( - ('relu{:d}'.format(i), nn.ReLU(inplace=True))) - attconv_layers.append(('conv_out', nn.Conv2d(kernel_size=1, in_channels=56, out_channels=1))) - self.attconv = nn.Sequential(OrderedDict(attconv_layers)) - else: - raise ValueError('Model not valid: {}'.format(model)) - - def get_attention(self, x: torch.Tensor) -> torch.Tensor: - - # Placeholder - att = None - - # Stem - x = self._swish(self._bn0(self._conv_stem(x))) - - # Blocks - for idx, block in enumerate(self._blocks): - drop_connect_rate = self._global_params.drop_connect_rate - if drop_connect_rate: - drop_connect_rate *= float(idx) / len(self._blocks) - x = block(x, drop_connect_rate=drop_connect_rate) - if idx == self.att_block_idx: - att = torch.sigmoid(self.attconv(x)) - break - - return att - - def extract_features(self, x: torch.Tensor) -> torch.Tensor: - # Stem - x = self._swish(self._bn0(self._conv_stem(x))) - - # Blocks - for idx, block in enumerate(self._blocks): - drop_connect_rate = self._global_params.drop_connect_rate - if drop_connect_rate: - drop_connect_rate *= float(idx) / len(self._blocks) - x = block(x, drop_connect_rate=drop_connect_rate) - if idx == self.att_block_idx: - att = torch.sigmoid(self.attconv(x)) - x = x * att - - # Head - x = self._swish(self._bn1(self._conv_head(x))) - - return x - - -class EfficientNetGenAutoAtt(FeatureExtractor): - def __init__(self, model: str, width: int): - super(EfficientNetGenAutoAtt, self).__init__() - - self.efficientnet = EfficientNetAutoAtt.from_pretrained(model) - self.efficientnet.init_att(model, width) - self.classifier = nn.Linear(self.efficientnet._conv_head.out_channels, 1) - del self.efficientnet._fc - - def features(self, x: torch.Tensor) -> torch.Tensor: - x = self.efficientnet.extract_features(x) - x = self.efficientnet._avg_pooling(x) - x = x.flatten(start_dim=1) - return x - - def forward(self, x): - x = self.features(x) - x = self.efficientnet._dropout(x) - x = self.classifier(x) - return x - - def get_attention(self, x: torch.Tensor) -> torch.Tensor: - return self.efficientnet.get_attention(x) - - -class EfficientNetAutoAttB4(EfficientNetGenAutoAtt): - def __init__(self): - super(EfficientNetAutoAttB4, self).__init__(model='efficientnet-b4', width=0) - - -""" -Xception -""" - - -class Xception(FeatureExtractor): - def __init__(self): - super(Xception, self).__init__() - self.xception = externals.xception() - self.xception.last_linear = nn.Linear(2048, 1) - - def features(self, x: torch.Tensor) -> torch.Tensor: - x = self.xception.features(x) - x = nn.ReLU(inplace=True)(x) - x = F.adaptive_avg_pool2d(x, (1, 1)) - x = x.view(x.size(0), -1) - return x - - def forward(self, x: torch.Tensor) -> torch.Tensor: - return self.xception.forward(x) - - -""" -Siamese tuning -""" - - -class SiameseTuning(FeatureExtractor): - def __init__(self, feat_ext: FeatureExtractor, num_feat: int, lastonly: bool = True): - super(SiameseTuning, self).__init__() - self.feat_ext = feat_ext() - if not hasattr(self.feat_ext, 'features'): - raise NotImplementedError('The provided feature extractor needs to provide a features() method') - self.lastonly = lastonly - self.classifier = nn.Sequential( - nn.BatchNorm1d(num_features=num_feat), - nn.Linear(in_features=num_feat, out_features=1), - ) - - def features(self, x): - x = self.feat_ext.features(x) - return x - - def forward(self, x: torch.Tensor) -> torch.Tensor: - if self.lastonly: - with torch.no_grad(): - x = self.features(x) - else: - x = self.features(x) - x = self.classifier(x) - return x - - def get_trainable_parameters(self): - if self.lastonly: - return self.classifier.parameters() - else: - return self.parameters() - - -class EfficientNetB4ST(SiameseTuning): - def __init__(self): - super(EfficientNetB4ST, self).__init__(feat_ext=EfficientNetB4, num_feat=1792, lastonly=True) - - -class EfficientNetAutoAttB4ST(SiameseTuning): - def __init__(self): - super(EfficientNetAutoAttB4ST, self).__init__(feat_ext=EfficientNetAutoAttB4, num_feat=1792, lastonly=True) - - -class XceptionST(SiameseTuning): - def __init__(self): - super(XceptionST, self).__init__(feat_ext=Xception, num_feat=2048, lastonly=True) diff --git a/models/icpr2020dfdc/architectures/tripletnet.py b/models/icpr2020dfdc/architectures/tripletnet.py deleted file mode 100644 index ae265322ce3ec97538113c8d7a79b3d1c7ef02f1..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/architectures/tripletnet.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -from . import fornet -from .fornet import FeatureExtractor - - -class TripletNet(FeatureExtractor): - """ - Template class for triplet net - """ - - def __init__(self, feat_ext: FeatureExtractor): - super(TripletNet, self).__init__() - self.feat_ext = feat_ext() - if not hasattr(self.feat_ext, 'features'): - raise NotImplementedError('The provided feature extractor needs to provide a features() method') - - def features(self, x): - return self.feat_ext.features(x) - - def forward(self, x1, x2, x3): - x1 = self.features(x1) - x2 = self.features(x2) - x3 = self.features(x3) - return x1, x2, x3 - - -class EfficientNetB4(TripletNet): - def __init__(self): - super(EfficientNetB4, self).__init__(feat_ext=fornet.EfficientNetB4) - - -class EfficientNetAutoAttB4(TripletNet): - def __init__(self): - super(EfficientNetAutoAttB4, self).__init__(feat_ext=fornet.EfficientNetAutoAttB4) diff --git a/models/icpr2020dfdc/architectures/weights.py b/models/icpr2020dfdc/architectures/weights.py deleted file mode 100644 index 0d2ee91a5f49f206ae370b1653497fb34970b982..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/architectures/weights.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" - -weight_url = { -'EfficientNetAutoAttB4ST_DFDC':'https://f002.backblazeb2.com/file/icpr2020/EfficientNetAutoAttB4ST_DFDC_bestval-4df0ef7d2f380a5955affa78c35d0942ac1cd65229510353b252737775515a33.pth', -'EfficientNetAutoAttB4ST_FFPP':'https://f002.backblazeb2.com/file/icpr2020/EfficientNetAutoAttB4ST_FFPP_bestval-ddb357503b9b902e1b925c2550415604c4252b9b9ecafeb7369dc58cc16e9edd.pth', -'EfficientNetAutoAttB4_DFDC':'https://f002.backblazeb2.com/file/icpr2020/EfficientNetAutoAttB4_DFDC_bestval-72ed969b2a395fffe11a0d5bf0a635e7260ba2588c28683630d97ff7153389fc.pth', -'EfficientNetAutoAttB4_FFPP':'https://f002.backblazeb2.com/file/icpr2020/EfficientNetAutoAttB4_FFPP_bestval-b0c9e9522a7143cf119843e910234be5e30f77dc527b1b427cdffa5ce3bdbc25.pth', -'EfficientNetB4ST_DFDC':'https://f002.backblazeb2.com/file/icpr2020/EfficientNetB4ST_DFDC_bestval-86f0a0701b18694dfb5e7837bd09fa8e48a5146c193227edccf59f1b038181c6.pth', -'EfficientNetB4ST_FFPP':'https://f002.backblazeb2.com/file/icpr2020/EfficientNetB4ST_FFPP_bestval-ccd016668071be5bf5fff68e446d055441739ec7113fb1a6eee998f08396ae92.pth', -'EfficientNetB4_DFDC':'https://f002.backblazeb2.com/file/icpr2020/EfficientNetB4_DFDC_bestval-c9f3663e2116d3356d056a0ce6453e0fc412a8df68ebd0902f07104d9129a09a.pth', -'EfficientNetB4_FFPP':'https://f002.backblazeb2.com/file/icpr2020/EfficientNetB4_FFPP_bestval-93aaad84946829e793d1a67ed7e0309b535e2f2395acb4f8d16b92c0616ba8d7.pth', -'Xception_DFDC':'https://f002.backblazeb2.com/file/icpr2020/Xception_DFDC_bestval-e826cdb64d73ef491e6b8ff8fce0e1e1b7fc1d8e2715bc51a56280fff17596f9.pth', -'Xception_FFPP':'https://f002.backblazeb2.com/file/icpr2020/Xception_FFPP_bestval-bb119e4913cb8f816cd28a03f81f4c603d6351bf8e3f8e3eb99eebc923aecd22.pth', -} \ No newline at end of file diff --git a/models/icpr2020dfdc/blazeface/__init__.py b/models/icpr2020dfdc/blazeface/__init__.py deleted file mode 100644 index aa716adf1c4eb6a8941605bb98506beeacfe24b2..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/blazeface/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .blazeface import BlazeFace -from .face_extract import FaceExtractor -from .read_video import VideoReader diff --git a/models/icpr2020dfdc/blazeface/anchors.npy b/models/icpr2020dfdc/blazeface/anchors.npy deleted file mode 100644 index 3ba12474802adea36a8e37ca648d46556cd35c92..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/blazeface/anchors.npy +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a10bb2fb93ab54ca426d6c750bfc3aad685028a16dcf231357d03694f261fd95 -size 28800 diff --git a/models/icpr2020dfdc/blazeface/blazeface.pth b/models/icpr2020dfdc/blazeface/blazeface.pth deleted file mode 100644 index e3ab57c26d4f6b2863b7f2019a6a352695b6d193..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/blazeface/blazeface.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:54ecff653feaaaf1f7d44b6aff28fd2fc50e483a4e847563b6dd261369c43ba4 -size 420224 diff --git a/models/icpr2020dfdc/blazeface/blazeface.py b/models/icpr2020dfdc/blazeface/blazeface.py deleted file mode 100644 index 302478072939a798a7558756f63d48a13619120f..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/blazeface/blazeface.py +++ /dev/null @@ -1,417 +0,0 @@ -from typing import List - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F - - -class BlazeBlock(nn.Module): - def __init__(self, in_channels, out_channels, kernel_size=3, stride=1): - super(BlazeBlock, self).__init__() - - self.stride = stride - self.channel_pad = out_channels - in_channels - - # TFLite uses slightly different padding than PyTorch - # on the depthwise conv layer when the stride is 2. - if stride == 2: - self.max_pool = nn.MaxPool2d(kernel_size=stride, stride=stride) - padding = 0 - else: - padding = (kernel_size - 1) // 2 - - self.convs = nn.Sequential( - nn.Conv2d(in_channels=in_channels, out_channels=in_channels, - kernel_size=kernel_size, stride=stride, padding=padding, - groups=in_channels, bias=True), - nn.Conv2d(in_channels=in_channels, out_channels=out_channels, - kernel_size=1, stride=1, padding=0, bias=True), - ) - - self.act = nn.ReLU(inplace=True) - - def forward(self, x): - if self.stride == 2: - h = F.pad(x, (0, 2, 0, 2), "constant", 0) - x = self.max_pool(x) - else: - h = x - - if self.channel_pad > 0: - x = F.pad(x, (0, 0, 0, 0, 0, self.channel_pad), "constant", 0) - - return self.act(self.convs(h) + x) - - -class BlazeFace(nn.Module): - """The BlazeFace face detection model from MediaPipe. - - The version from MediaPipe is simpler than the one in the paper; - it does not use the "double" BlazeBlocks. - - Because we won't be training this model, it doesn't need to have - batchnorm layers. These have already been "folded" into the conv - weights by TFLite. - - The conversion to PyTorch is fairly straightforward, but there are - some small differences between TFLite and PyTorch in how they handle - padding on conv layers with stride 2. - - This version works on batches, while the MediaPipe version can only - handle a single image at a time. - - Based on code from https://github.com/tkat0/PyTorch_BlazeFace/ and - https://github.com/google/mediapipe/ - """ - input_size = (128, 128) - - detection_keys = [ - 'ymin', 'xmin', 'ymax', 'xmax', - 'kp1x', 'kp1y', 'kp2x', 'kp2y', 'kp3x', 'kp3y', 'kp4x', 'kp4y', 'kp5x', 'kp5y', 'kp6x', 'kp6y', - 'conf' - ] - - def __init__(self): - super(BlazeFace, self).__init__() - - # These are the settings from the MediaPipe example graph - # mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt - self.num_classes = 1 - self.num_anchors = 896 - self.num_coords = 16 - self.score_clipping_thresh = 100.0 - self.x_scale = 128.0 - self.y_scale = 128.0 - self.h_scale = 128.0 - self.w_scale = 128.0 - self.min_score_thresh = 0.75 - self.min_suppression_threshold = 0.3 - - self._define_layers() - - def _define_layers(self): - self.backbone1 = nn.Sequential( - nn.Conv2d(in_channels=3, out_channels=24, kernel_size=5, stride=2, padding=0, bias=True), - nn.ReLU(inplace=True), - - BlazeBlock(24, 24), - BlazeBlock(24, 28), - BlazeBlock(28, 32, stride=2), - BlazeBlock(32, 36), - BlazeBlock(36, 42), - BlazeBlock(42, 48, stride=2), - BlazeBlock(48, 56), - BlazeBlock(56, 64), - BlazeBlock(64, 72), - BlazeBlock(72, 80), - BlazeBlock(80, 88), - ) - - self.backbone2 = nn.Sequential( - BlazeBlock(88, 96, stride=2), - BlazeBlock(96, 96), - BlazeBlock(96, 96), - BlazeBlock(96, 96), - BlazeBlock(96, 96), - ) - - self.classifier_8 = nn.Conv2d(88, 2, 1, bias=True) - self.classifier_16 = nn.Conv2d(96, 6, 1, bias=True) - - self.regressor_8 = nn.Conv2d(88, 32, 1, bias=True) - self.regressor_16 = nn.Conv2d(96, 96, 1, bias=True) - - def forward(self, x): - # TFLite uses slightly different padding on the first conv layer - # than PyTorch, so do it manually. - x = F.pad(x, (1, 2, 1, 2), "constant", 0) - - b = x.shape[0] # batch size, needed for reshaping later - - x = self.backbone1(x) # (b, 88, 16, 16) - h = self.backbone2(x) # (b, 96, 8, 8) - - # Note: Because PyTorch is NCHW but TFLite is NHWC, we need to - # permute the output from the conv layers before reshaping it. - - c1 = self.classifier_8(x) # (b, 2, 16, 16) - c1 = c1.permute(0, 2, 3, 1) # (b, 16, 16, 2) - c1 = c1.reshape(b, -1, 1) # (b, 512, 1) - - c2 = self.classifier_16(h) # (b, 6, 8, 8) - c2 = c2.permute(0, 2, 3, 1) # (b, 8, 8, 6) - c2 = c2.reshape(b, -1, 1) # (b, 384, 1) - - c = torch.cat((c1, c2), dim=1) # (b, 896, 1) - - r1 = self.regressor_8(x) # (b, 32, 16, 16) - r1 = r1.permute(0, 2, 3, 1) # (b, 16, 16, 32) - r1 = r1.reshape(b, -1, 16) # (b, 512, 16) - - r2 = self.regressor_16(h) # (b, 96, 8, 8) - r2 = r2.permute(0, 2, 3, 1) # (b, 8, 8, 96) - r2 = r2.reshape(b, -1, 16) # (b, 384, 16) - - r = torch.cat((r1, r2), dim=1) # (b, 896, 16) - return [r, c] - - def _device(self): - """Which device (CPU or GPU) is being used by this model?""" - return self.classifier_8.weight.device - - def load_weights(self, path): - self.load_state_dict(torch.load(path)) - self.eval() - - def load_anchors(self, path): - self.anchors = torch.tensor(np.load(path), dtype=torch.float32, device=self._device()) - assert (self.anchors.ndimension() == 2) - assert (self.anchors.shape[0] == self.num_anchors) - assert (self.anchors.shape[1] == 4) - - def _preprocess(self, x): - """Converts the image pixels to the range [-1, 1].""" - return x.float() / 127.5 - 1.0 - - def predict_on_image(self, img): - """Makes a prediction on a single image. - - Arguments: - img: a NumPy array of shape (H, W, 3) or a PyTorch tensor of - shape (3, H, W). The image's height and width should be - 128 pixels. - - Returns: - A tensor with face detections. - """ - if isinstance(img, np.ndarray): - img = torch.from_numpy(img).permute((2, 0, 1)) - - return self.predict_on_batch(img.unsqueeze(0))[0] - - def predict_on_batch(self, x: np.ndarray or torch.Tensor, apply_nms: bool = True) -> List[torch.Tensor]: - """Makes a prediction on a batch of images. - - Arguments: - x: a NumPy array of shape (b, H, W, 3) or a PyTorch tensor of - shape (b, 3, H, W). The height and width should be 128 pixels. - apply_nms: pass False to not apply non-max suppression - - Returns: - A list containing a tensor of face detections for each image in - the batch. If no faces are found for an image, returns a tensor - of shape (0, 17). - - Each face detection is a PyTorch tensor consisting of 17 numbers: - - ymin, xmin, ymax, xmax - - x,y-coordinates for the 6 keypoints - - confidence score - """ - if isinstance(x, np.ndarray): - x = torch.from_numpy(x).permute((0, 3, 1, 2)) - - assert x.shape[1] == 3 - assert x.shape[2] == 128 - assert x.shape[3] == 128 - - # 1. Preprocess the images into tensors: - x = x.to(self._device()) - x = self._preprocess(x) - - # 2. Run the neural network: - with torch.no_grad(): - out: torch.Tensor = self.__call__(x) - - # 3. Postprocess the raw predictions: - detections = self._tensors_to_detections(out[0], out[1], self.anchors) - - # 4. Non-maximum suppression to remove overlapping detections: - return self.nms(detections) if apply_nms else detections - - def nms(self, detections: List[torch.Tensor]) -> List[torch.Tensor]: - """Filters out overlapping detections.""" - filtered_detections = [] - for i in range(len(detections)): - faces = self._weighted_non_max_suppression(detections[i]) - faces = torch.stack(faces) if len(faces) > 0 else torch.zeros((0, 17), device=self._device()) - filtered_detections.append(faces) - - return filtered_detections - - def _tensors_to_detections(self, raw_box_tensor: torch.Tensor, raw_score_tensor: torch.Tensor, anchors) -> List[ - torch.Tensor]: - """The output of the neural network is a tensor of shape (b, 896, 16) - containing the bounding box regressor predictions, as well as a tensor - of shape (b, 896, 1) with the classification confidences. - - This function converts these two "raw" tensors into proper detections. - Returns a list of (num_detections, 17) tensors, one for each image in - the batch. - - This is based on the source code from: - mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc - mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto - """ - assert raw_box_tensor.ndimension() == 3 - assert raw_box_tensor.shape[1] == self.num_anchors - assert raw_box_tensor.shape[2] == self.num_coords - - assert raw_score_tensor.ndimension() == 3 - assert raw_score_tensor.shape[1] == self.num_anchors - assert raw_score_tensor.shape[2] == self.num_classes - - assert raw_box_tensor.shape[0] == raw_score_tensor.shape[0] - - detection_boxes = self._decode_boxes(raw_box_tensor, anchors) - - thresh = self.score_clipping_thresh - raw_score_tensor = raw_score_tensor.clamp(-thresh, thresh) - detection_scores = raw_score_tensor.sigmoid().squeeze(dim=-1) - - # Note: we stripped off the last dimension from the scores tensor - # because there is only has one class. Now we can simply use a mask - # to filter out the boxes with too low confidence. - mask = detection_scores >= self.min_score_thresh - - # Because each image from the batch can have a different number of - # detections, process them one at a time using a loop. - output_detections = [] - for i in range(raw_box_tensor.shape[0]): - boxes = detection_boxes[i, mask[i]] - scores = detection_scores[i, mask[i]].unsqueeze(dim=-1) - output_detections.append(torch.cat((boxes, scores), dim=-1)) - - return output_detections - - def _decode_boxes(self, raw_boxes, anchors): - """Converts the predictions into actual coordinates using - the anchor boxes. Processes the entire batch at once. - """ - boxes = torch.zeros_like(raw_boxes) - - x_center = raw_boxes[..., 0] / self.x_scale * anchors[:, 2] + anchors[:, 0] - y_center = raw_boxes[..., 1] / self.y_scale * anchors[:, 3] + anchors[:, 1] - - w = raw_boxes[..., 2] / self.w_scale * anchors[:, 2] - h = raw_boxes[..., 3] / self.h_scale * anchors[:, 3] - - boxes[..., 0] = y_center - h / 2. # ymin - boxes[..., 1] = x_center - w / 2. # xmin - boxes[..., 2] = y_center + h / 2. # ymax - boxes[..., 3] = x_center + w / 2. # xmax - - for k in range(6): - offset = 4 + k * 2 - keypoint_x = raw_boxes[..., offset] / self.x_scale * anchors[:, 2] + anchors[:, 0] - keypoint_y = raw_boxes[..., offset + 1] / self.y_scale * anchors[:, 3] + anchors[:, 1] - boxes[..., offset] = keypoint_x - boxes[..., offset + 1] = keypoint_y - - return boxes - - def _weighted_non_max_suppression(self, detections): - """The alternative NMS method as mentioned in the BlazeFace paper: - - "We replace the suppression algorithm with a blending strategy that - estimates the regression parameters of a bounding box as a weighted - mean between the overlapping predictions." - - The original MediaPipe code assigns the score of the most confident - detection to the weighted detection, but we take the average score - of the overlapping detections. - - The input detections should be a Tensor of shape (count, 17). - - Returns a list of PyTorch tensors, one for each detected face. - - This is based on the source code from: - mediapipe/calculators/util/non_max_suppression_calculator.cc - mediapipe/calculators/util/non_max_suppression_calculator.proto - """ - if len(detections) == 0: return [] - - output_detections = [] - - # Sort the detections from highest to lowest score. - remaining = torch.argsort(detections[:, 16], descending=True) - - while len(remaining) > 0: - detection = detections[remaining[0]] - - # Compute the overlap between the first box and the other - # remaining boxes. (Note that the other_boxes also include - # the first_box.) - first_box = detection[:4] - other_boxes = detections[remaining, :4] - ious = overlap_similarity(first_box, other_boxes) - - # If two detections don't overlap enough, they are considered - # to be from different faces. - mask = ious > self.min_suppression_threshold - overlapping = remaining[mask] - remaining = remaining[~mask] - - # Take an average of the coordinates from the overlapping - # detections, weighted by their confidence scores. - weighted_detection = detection.clone() - if len(overlapping) > 1: - coordinates = detections[overlapping, :16] - scores = detections[overlapping, 16:17] - total_score = scores.sum() - weighted = (coordinates * scores).sum(dim=0) / total_score - weighted_detection[:16] = weighted - weighted_detection[16] = total_score / len(overlapping) - - output_detections.append(weighted_detection) - - return output_detections - - # IOU code from https://github.com/amdegroot/ssd.pytorch/blob/master/layers/box_utils.py - - -def intersect(box_a, box_b): - """ We resize both tensors to [A,B,2] without new malloc: - [A,2] -> [A,1,2] -> [A,B,2] - [B,2] -> [1,B,2] -> [A,B,2] - Then we compute the area of intersect between box_a and box_b. - Args: - box_a: (tensor) bounding boxes, Shape: [A,4]. - box_b: (tensor) bounding boxes, Shape: [B,4]. - Return: - (tensor) intersection area, Shape: [A,B]. - """ - A = box_a.size(0) - B = box_b.size(0) - max_xy = torch.min(box_a[:, 2:].unsqueeze(1).expand(A, B, 2), - box_b[:, 2:].unsqueeze(0).expand(A, B, 2)) - min_xy = torch.max(box_a[:, :2].unsqueeze(1).expand(A, B, 2), - box_b[:, :2].unsqueeze(0).expand(A, B, 2)) - inter = torch.clamp((max_xy - min_xy), min=0) - return inter[:, :, 0] * inter[:, :, 1] - - -def jaccard(box_a, box_b): - """Compute the jaccard overlap of two sets of boxes. The jaccard overlap - is simply the intersection over union of two boxes. Here we operate on - ground truth boxes and default boxes. - E.g.: - A ∩ B / A ∪ B = A ∩ B / (area(A) + area(B) - A ∩ B) - Args: - box_a: (tensor) Ground truth bounding boxes, Shape: [num_objects,4] - box_b: (tensor) Prior boxes from priorbox layers, Shape: [num_priors,4] - Return: - jaccard overlap: (tensor) Shape: [box_a.size(0), box_b.size(0)] - """ - inter = intersect(box_a, box_b) - area_a = ((box_a[:, 2] - box_a[:, 0]) * - (box_a[:, 3] - box_a[:, 1])).unsqueeze(1).expand_as(inter) # [A,B] - area_b = ((box_b[:, 2] - box_b[:, 0]) * - (box_b[:, 3] - box_b[:, 1])).unsqueeze(0).expand_as(inter) # [A,B] - union = area_a + area_b - inter - return inter / union # [A,B] - - -def overlap_similarity(box, other_boxes): - """Computes the IOU between a bounding box and set of other boxes.""" - return jaccard(box.unsqueeze(0), other_boxes).squeeze(0) diff --git a/models/icpr2020dfdc/blazeface/face_extract.py b/models/icpr2020dfdc/blazeface/face_extract.py deleted file mode 100644 index cece5d650f4f0ac115a26f5e32cc55d8ef159303..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/blazeface/face_extract.py +++ /dev/null @@ -1,470 +0,0 @@ -import os -from typing import Tuple, List - -import cv2 -import numpy as np -import torch -from PIL import Image - -from blazeface import BlazeFace - - -class FaceExtractor: - """Wrapper for face extraction workflow.""" - - def __init__(self, video_read_fn = None, facedet: BlazeFace = None): - """Creates a new FaceExtractor. - - Arguments: - video_read_fn: a function that takes in a path to a video file - and returns a tuple consisting of a NumPy array with shape - (num_frames, H, W, 3) and a list of frame indices, or None - in case of an error - facedet: the face detector object - """ - self.video_read_fn = video_read_fn - self.facedet = facedet - - def process_image(self, path: str = None, img: Image.Image or np.ndarray = None) -> dict: - """ - Process a single image - :param path: Path to the image - :param img: image - :return: - """ - - if img is not None and path is not None: - raise ValueError('Only one argument between path and img can be specified') - if img is None and path is None: - raise ValueError('At least one argument between path and img must be specified') - - target_size = self.facedet.input_size - - if img is None: - img = np.asarray(Image.open(str(path))) - else: - img = np.asarray(img) - - # Split the frames into several tiles. Resize the tiles to 128x128. - tiles, resize_info = self._tile_frames(np.expand_dims(img, 0), target_size) - # tiles has shape (num_tiles, target_size, target_size, 3) - # resize_info is a list of four elements [resize_factor_y, resize_factor_x, 0, 0] - - # Run the face detector. The result is a list of PyTorch tensors, - # one for each tile in the batch. - detections = self.facedet.predict_on_batch(tiles, apply_nms=False) - - # Convert the detections from 128x128 back to the original frame size. - detections = self._resize_detections(detections, target_size, resize_info) - - # Because we have several tiles for each frame, combine the predictions - # from these tiles. The result is a list of PyTorch tensors, but now one - # for each frame (rather than each tile). - num_frames = 1 - frame_size = (img.shape[1], img.shape[0]) - detections = self._untile_detections(num_frames, frame_size, detections) - - # The same face may have been detected in multiple tiles, so filter out - # overlapping detections. This is done separately for each frame. - detections = self.facedet.nms(detections) - - # Crop the faces out of the original frame. - frameref_detections = self._add_margin_to_detections(detections[0], frame_size, 0.2) - faces = self._crop_faces(img, frameref_detections) - kpts = self._crop_kpts(img, detections[0], 0.3) - - # Add additional information about the frame and detections. - scores = list(detections[0][:, 16].cpu().numpy()) - frame_dict = {"frame_w": frame_size[0], - "frame_h": frame_size[1], - "faces": faces, - "kpts": kpts, - "detections": frameref_detections.cpu().numpy(), - "scores": scores, - } - - # Sort faces by descending confidence - frame_dict = self._soft_faces_by_descending_score(frame_dict) - - return frame_dict - - def _soft_faces_by_descending_score(self, frame_dict: dict) -> dict: - if len(frame_dict['scores']) > 1: - sort_idxs = np.argsort(frame_dict['scores'])[::-1] - new_faces = [frame_dict['faces'][i] for i in sort_idxs] - new_kpts = [frame_dict['kpts'][i] for i in sort_idxs] - new_detections = frame_dict['detections'][sort_idxs] - new_scores = [frame_dict['scores'][i] for i in sort_idxs] - frame_dict['faces'] = new_faces - frame_dict['kpts'] = new_kpts - frame_dict['detections'] = new_detections - frame_dict['scores'] = new_scores - return frame_dict - - def process_videos(self, input_dir, filenames, video_idxs) -> List[dict]: - """For the specified selection of videos, grabs one or more frames - from each video, runs the face detector, and tries to find the faces - in each frame. - - The frames are split into tiles, and the tiles from the different videos - are concatenated into a single batch. This means the face detector gets - a batch of size len(video_idxs) * num_frames * num_tiles (usually 3). - - Arguments: - input_dir: base folder where the video files are stored - filenames: list of all video files in the input_dir - video_idxs: one or more indices from the filenames list; these - are the videos we'll actually process - - Returns a list of dictionaries, one for each frame read from each video. - - This dictionary contains: - - video_idx: the video this frame was taken from - - frame_idx: the index of the frame in the video - - frame_w, frame_h: original dimensions of the frame - - faces: a list containing zero or more NumPy arrays with a face crop - - scores: a list array with the confidence score for each face crop - - If reading a video failed for some reason, it will not appear in the - output array. Note that there's no guarantee a given video will actually - have num_frames results (as soon as a reading problem is encountered for - a video, we continue with the next video). - """ - target_size = self.facedet.input_size - - videos_read = [] - frames_read = [] - frames = [] - tiles = [] - resize_info = [] - - for video_idx in video_idxs: - # Read the full-size frames from this video. - filename = filenames[video_idx] - video_path = os.path.join(input_dir, filename) - result = self.video_read_fn(video_path) - - # Error? Then skip this video. - if result is None: continue - - videos_read.append(video_idx) - - # Keep track of the original frames (need them later). - my_frames, my_idxs = result - frames.append(my_frames) - frames_read.append(my_idxs) - - # Split the frames into several tiles. Resize the tiles to 128x128. - my_tiles, my_resize_info = self._tile_frames(my_frames, target_size) - tiles.append(my_tiles) - resize_info.append(my_resize_info) - - if len(tiles) == 0: - return [] - # Put all the tiles for all the frames from all the videos into - # a single batch. - batch = np.concatenate(tiles) - - # Run the face detector. The result is a list of PyTorch tensors, - # one for each image in the batch. - all_detections = self.facedet.predict_on_batch(batch, apply_nms=False) - - result = [] - offs = 0 - for v in range(len(tiles)): - # Not all videos may have the same number of tiles, so find which - # detections go with which video. - num_tiles = tiles[v].shape[0] - detections = all_detections[offs:offs + num_tiles] - offs += num_tiles - - # Convert the detections from 128x128 back to the original frame size. - detections = self._resize_detections(detections, target_size, resize_info[v]) - - # Because we have several tiles for each frame, combine the predictions - # from these tiles. The result is a list of PyTorch tensors, but now one - # for each frame (rather than each tile). - num_frames = frames[v].shape[0] - frame_size = (frames[v].shape[2], frames[v].shape[1]) - detections = self._untile_detections(num_frames, frame_size, detections) - - # The same face may have been detected in multiple tiles, so filter out - # overlapping detections. This is done separately for each frame. - detections = self.facedet.nms(detections) - - for i in range(len(detections)): - # Crop the faces out of the original frame. - frameref_detections = self._add_margin_to_detections(detections[i], frame_size, 0.2) - faces = self._crop_faces(frames[v][i], frameref_detections) - kpts = self._crop_kpts(frames[v][i], detections[i], 0.3) - - # Add additional information about the frame and detections. - scores = list(detections[i][:, 16].cpu().numpy()) - frame_dict = {"video_idx": videos_read[v], - "frame_idx": frames_read[v][i], - "frame_w": frame_size[0], - "frame_h": frame_size[1], - "frame": frames[v][i], - "faces": faces, - "kpts": kpts, - "detections": frameref_detections.cpu().numpy(), - "scores": scores, - } - # Sort faces by descending confidence - frame_dict = self._soft_faces_by_descending_score(frame_dict) - - result.append(frame_dict) - - return result - - def process_video(self, video_path): - """Convenience method for doing face extraction on a single video.""" - input_dir = os.path.dirname(video_path) - filenames = [os.path.basename(video_path)] - return self.process_videos(input_dir, filenames, [0]) - - def _tile_frames(self, frames: np.ndarray, target_size: Tuple[int, int]) -> (np.ndarray, List[float]): - """Splits each frame into several smaller, partially overlapping tiles - and resizes each tile to target_size. - - After a bunch of experimentation, I found that for a 1920x1080 video, - BlazeFace works better on three 1080x1080 windows. These overlap by 420 - pixels. (Two windows also work but it's best to have a clean center crop - in there as well.) - - I also tried 6 windows of size 720x720 (horizontally: 720|360, 360|720; - vertically: 720|1200, 480|720|480, 1200|720) but that gives many false - positives when a window has no face in it. - - For a video in portrait orientation (1080x1920), we only take a single - crop of the top-most 1080 pixels. If we split up the video vertically, - then we might get false positives again. - - (NOTE: Not all videos are necessarily 1080p but the code can handle this.) - - Arguments: - frames: NumPy array of shape (num_frames, height, width, 3) - target_size: (width, height) - - Returns: - - a new (num_frames * N, target_size[1], target_size[0], 3) array - where N is the number of tiles used. - - a list [scale_w, scale_h, offset_x, offset_y] that describes how - to map the resized and cropped tiles back to the original image - coordinates. This is needed for scaling up the face detections - from the smaller image to the original image, so we can take the - face crops in the original coordinate space. - """ - num_frames, H, W, _ = frames.shape - - num_h, num_v, split_size, x_step, y_step = self.get_tiles_params(H, W) - - splits = np.zeros((num_frames * num_v * num_h, target_size[1], target_size[0], 3), dtype=np.uint8) - - i = 0 - for f in range(num_frames): - y = 0 - for v in range(num_v): - x = 0 - for h in range(num_h): - crop = frames[f, y:y + split_size, x:x + split_size, :] - splits[i] = cv2.resize(crop, target_size, interpolation=cv2.INTER_AREA) - x += x_step - i += 1 - y += y_step - - resize_info = [split_size / target_size[0], split_size / target_size[1], 0, 0] - return splits, resize_info - - def get_tiles_params(self, H, W): - split_size = min(H, W, 720) - x_step = (W - split_size) // 2 - y_step = (H - split_size) // 2 - num_v = (H - split_size) // y_step + 1 if y_step > 0 else 1 - num_h = (W - split_size) // x_step + 1 if x_step > 0 else 1 - return num_h, num_v, split_size, x_step, y_step - - def _resize_detections(self, detections, target_size, resize_info): - """Converts a list of face detections back to the original - coordinate system. - - Arguments: - detections: a list containing PyTorch tensors of shape (num_faces, 17) - target_size: (width, height) - resize_info: [scale_w, scale_h, offset_x, offset_y] - """ - projected = [] - target_w, target_h = target_size - scale_w, scale_h, offset_x, offset_y = resize_info - - for i in range(len(detections)): - detection = detections[i].clone() - - # ymin, xmin, ymax, xmax - for k in range(2): - detection[:, k * 2] = (detection[:, k * 2] * target_h - offset_y) * scale_h - detection[:, k * 2 + 1] = (detection[:, k * 2 + 1] * target_w - offset_x) * scale_w - - # keypoints are x,y - for k in range(2, 8): - detection[:, k * 2] = (detection[:, k * 2] * target_w - offset_x) * scale_w - detection[:, k * 2 + 1] = (detection[:, k * 2 + 1] * target_h - offset_y) * scale_h - - projected.append(detection) - - return projected - - def _untile_detections(self, num_frames: int, frame_size: Tuple[int, int], detections: List[torch.Tensor]) -> List[ - torch.Tensor]: - """With N tiles per frame, there also are N times as many detections. - This function groups together the detections for a given frame; it is - the complement to tile_frames(). - """ - combined_detections = [] - - W, H = frame_size - - num_h, num_v, split_size, x_step, y_step = self.get_tiles_params(H, W) - - i = 0 - for f in range(num_frames): - detections_for_frame = [] - y = 0 - for v in range(num_v): - x = 0 - for h in range(num_h): - # Adjust the coordinates based on the split positions. - detection = detections[i].clone() - if detection.shape[0] > 0: - for k in range(2): - detection[:, k * 2] += y - detection[:, k * 2 + 1] += x - for k in range(2, 8): - detection[:, k * 2] += x - detection[:, k * 2 + 1] += y - - detections_for_frame.append(detection) - x += x_step - i += 1 - y += y_step - - combined_detections.append(torch.cat(detections_for_frame)) - - return combined_detections - - def _add_margin_to_detections(self, detections: torch.Tensor, frame_size: Tuple[int, int], - margin: float = 0.2) -> torch.Tensor: - """Expands the face bounding box. - - NOTE: The face detections often do not include the forehead, which - is why we use twice the margin for ymin. - - Arguments: - detections: a PyTorch tensor of shape (num_detections, 17) - frame_size: maximum (width, height) - margin: a percentage of the bounding box's height - - Returns a PyTorch tensor of shape (num_detections, 17). - """ - offset = torch.round(margin * (detections[:, 2] - detections[:, 0])) - detections = detections.clone() - detections[:, 0] = torch.clamp(detections[:, 0] - offset * 2, min=0) # ymin - detections[:, 1] = torch.clamp(detections[:, 1] - offset, min=0) # xmin - detections[:, 2] = torch.clamp(detections[:, 2] + offset, max=frame_size[1]) # ymax - detections[:, 3] = torch.clamp(detections[:, 3] + offset, max=frame_size[0]) # xmax - return detections - - def _crop_faces(self, frame: np.ndarray, detections: torch.Tensor) -> List[np.ndarray]: - """Copies the face region(s) from the given frame into a set - of new NumPy arrays. - - Arguments: - frame: a NumPy array of shape (H, W, 3) - detections: a PyTorch tensor of shape (num_detections, 17) - - Returns a list of NumPy arrays, one for each face crop. If there - are no faces detected for this frame, returns an empty list. - """ - faces = [] - for i in range(len(detections)): - ymin, xmin, ymax, xmax = detections[i, :4].cpu().numpy().astype(int) - face = frame[ymin:ymax, xmin:xmax, :] - faces.append(face) - return faces - - def _crop_kpts(self, frame: np.ndarray, detections: torch.Tensor, face_fraction: float): - """Copies the parts region(s) from the given frame into a set - of new NumPy arrays. - - Arguments: - frame: a NumPy array of shape (H, W, 3) - detections: a PyTorch tensor of shape (num_detections, 17) - face_fraction: float between 0 and 1 indicating how big are the parts to be extracted w.r.t the whole face - - Returns a list of NumPy arrays, one for each face crop. If there - are no faces detected for this frame, returns an empty list. - """ - faces = [] - for i in range(len(detections)): - kpts = [] - size = int(face_fraction * min(detections[i, 2] - detections[i, 0], detections[i, 3] - detections[i, 1])) - kpts_coords = detections[i, 4:16].cpu().numpy().astype(int) - for kpidx in range(6): - kpx, kpy = kpts_coords[kpidx * 2:kpidx * 2 + 2] - kpt = frame[kpy - size // 2:kpy - size // 2 + size, kpx - size // 2:kpx - size // 2 + size, ] - kpts.append(kpt) - faces.append(kpts) - return faces - - def remove_large_crops(self, crops, pct=0.1): - """Removes faces from the results if they take up more than X% - of the video. Such a face is likely a false positive. - - This is an optional postprocessing step. Modifies the original - data structure. - - Arguments: - crops: a list of dictionaries with face crop data - pct: maximum portion of the frame a crop may take up - """ - for i in range(len(crops)): - frame_data = crops[i] - video_area = frame_data["frame_w"] * frame_data["frame_h"] - faces = frame_data["faces"] - scores = frame_data["scores"] - new_faces = [] - new_scores = [] - for j in range(len(faces)): - face = faces[j] - face_H, face_W, _ = face.shape - face_area = face_H * face_W - if face_area / video_area < 0.1: - new_faces.append(face) - new_scores.append(scores[j]) - frame_data["faces"] = new_faces - frame_data["scores"] = new_scores - - def keep_only_best_face(self, crops): - """For each frame, only keeps the face with the highest confidence. - - This gets rid of false positives, but obviously is problematic for - videos with two people! - - This is an optional postprocessing step. Modifies the original - data structure. - """ - for i in range(len(crops)): - frame_data = crops[i] - if len(frame_data["faces"]) > 0: - frame_data["faces"] = frame_data["faces"][:1] - frame_data["scores"] = frame_data["scores"][:1] - - # TODO: def filter_likely_false_positives(self, crops): - # if only some frames have more than 1 face, it's likely a false positive - # if most frames have more than 1 face, it's probably two people - # so find the % of frames with > 1 face; if > 0.X, keep the two best faces - - # TODO: def filter_by_score(self, crops, min_score) to remove any - # crops with a confidence score lower than min_score - - # TODO: def sort_by_histogram(self, crops) for videos with 2 people. diff --git a/models/icpr2020dfdc/blazeface/read_video.py b/models/icpr2020dfdc/blazeface/read_video.py deleted file mode 100644 index 19cb2420c77f3796abd9f6edf7cc8d8fe37c142a..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/blazeface/read_video.py +++ /dev/null @@ -1,213 +0,0 @@ -import cv2 -import numpy as np - - -class VideoReader: - """Helper class for reading one or more frames from a video file.""" - - def __init__(self, verbose=True, insets=(0, 0)): - """Creates a new VideoReader. - - Arguments: - verbose: whether to print warnings and error messages - insets: amount to inset the image by, as a percentage of - (width, height). This lets you "zoom in" to an image - to remove unimportant content around the borders. - Useful for face detection, which may not work if the - faces are too small. - """ - self.verbose = verbose - self.insets = insets - - def read_frames(self, path, num_frames, jitter=0, seed=None): - """Reads frames that are always evenly spaced throughout the video. - - Arguments: - path: the video file - num_frames: how many frames to read, -1 means the entire video - (warning: this will take up a lot of memory!) - jitter: if not 0, adds small random offsets to the frame indices; - this is useful so we don't always land on even or odd frames - seed: random seed for jittering; if you set this to a fixed value, - you probably want to set it only on the first video - """ - assert num_frames > 0 - - capture = cv2.VideoCapture(path) - frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) - if frame_count <= 0: return None - - frame_idxs = np.linspace(0, frame_count - 1, num_frames, endpoint=True, dtype=int) - frame_idxs = np.unique(frame_idxs) # Avoid repeating frame idxs otherwise it breaks reading - if jitter > 0: - np.random.seed(seed) - jitter_offsets = np.random.randint(-jitter, jitter, len(frame_idxs)) - frame_idxs = np.clip(frame_idxs + jitter_offsets, 0, frame_count - 1) - - result = self._read_frames_at_indices(path, capture, frame_idxs) - capture.release() - return result - - def read_random_frames(self, path, num_frames, seed=None): - """Picks the frame indices at random. - - Arguments: - path: the video file - num_frames: how many frames to read, -1 means the entire video - (warning: this will take up a lot of memory!) - """ - assert num_frames > 0 - np.random.seed(seed) - - capture = cv2.VideoCapture(path) - frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) - if frame_count <= 0: return None - - frame_idxs = sorted(np.random.choice(np.arange(0, frame_count), num_frames)) - result = self._read_frames_at_indices(path, capture, frame_idxs) - - capture.release() - return result - - def read_frames_at_indices(self, path, frame_idxs): - """Reads frames from a video and puts them into a NumPy array. - - Arguments: - path: the video file - frame_idxs: a list of frame indices. Important: should be - sorted from low-to-high! If an index appears multiple - times, the frame is still read only once. - - Returns: - - a NumPy array of shape (num_frames, height, width, 3) - - a list of the frame indices that were read - - Reading stops if loading a frame fails, in which case the first - dimension returned may actually be less than num_frames. - - Returns None if an exception is thrown for any reason, or if no - frames were read. - """ - assert len(frame_idxs) > 0 - capture = cv2.VideoCapture(path) - result = self._read_frames_at_indices(path, capture, frame_idxs) - capture.release() - return result - - def _read_frames_at_indices(self, path, capture, frame_idxs): - try: - frames = [] - idxs_read = [] - for frame_idx in range(frame_idxs[0], frame_idxs[-1] + 1): - # Get the next frame, but don't decode if we're not using it. - ret = capture.grab() - if not ret: - if self.verbose: - print("Error grabbing frame %d from movie %s" % (frame_idx, path)) - break - - # Need to look at this frame? - current = len(idxs_read) - if frame_idx == frame_idxs[current]: - ret, frame = capture.retrieve() - if not ret or frame is None: - if self.verbose: - print("Error retrieving frame %d from movie %s" % (frame_idx, path)) - break - - frame = self._postprocess_frame(frame) - frames.append(frame) - idxs_read.append(frame_idx) - - if len(frames) > 0: - return np.stack(frames), idxs_read - if self.verbose: - print("No frames read from movie %s" % path) - return None - except: - if self.verbose: - print("Exception while reading movie %s" % path) - return None - - def read_middle_frame(self, path): - """Reads the frame from the middle of the video.""" - capture = cv2.VideoCapture(path) - frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) - result = self._read_frame_at_index(path, capture, frame_count // 2) - capture.release() - return result - - def read_frame_at_index(self, path, frame_idx): - """Reads a single frame from a video. - - If you just want to read a single frame from the video, this is more - efficient than scanning through the video to find the frame. However, - for reading multiple frames it's not efficient. - - My guess is that a "streaming" approach is more efficient than a - "random access" approach because, unless you happen to grab a keyframe, - the decoder still needs to read all the previous frames in order to - reconstruct the one you're asking for. - - Returns a NumPy array of shape (1, H, W, 3) and the index of the frame, - or None if reading failed. - """ - capture = cv2.VideoCapture(path) - result = self._read_frame_at_index(path, capture, frame_idx) - capture.release() - return result - - def _read_frame_at_index(self, path, capture, frame_idx): - capture.set(cv2.CAP_PROP_POS_FRAMES, frame_idx) - ret, frame = capture.read() - if not ret or frame is None: - if self.verbose: - print("Error retrieving frame %d from movie %s" % (frame_idx, path)) - return None - else: - frame = self._postprocess_frame(frame) - return np.expand_dims(frame, axis=0), [frame_idx] - - def _postprocess_frame(self, frame): - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - - if self.insets[0] > 0: - W = frame.shape[1] - p = int(W * self.insets[0]) - frame = frame[:, p:-p, :] - - if self.insets[1] > 0: - H = frame.shape[1] - q = int(H * self.insets[1]) - frame = frame[q:-q, :, :] - - return frame - - -class VideoReaderIspl(VideoReader): - """ - Derived VideoReader class with overriden read_frames method - """ - - def read_frames_with_hop(self, path: str, num_frames: int = -1, fps: int = -1): - """Reads frames up to a certain number spaced throughout the video with a rate decided by the user. - - Arguments: - path: the video file - num_frames: how many frames to read, -1 means the entire video - (warning: this will take up a lot of memory!) - fps: how many frames per second to pick - """ - assert num_frames > 0 - - capture = cv2.VideoCapture(path) - frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) - if frame_count <= 0: return None - video_rate = capture.get(cv2.CAP_PROP_FPS) - hop = 1 if fps == -1 else max(video_rate // fps, 1) - end_pts = frame_count if num_frames == -1 else num_frames * hop - frame_idxs = np.arange(0, end_pts - 1, hop, endpoint=True, dtype=int) - - result = self._read_frames_at_indices(path, capture, frame_idxs) - capture.release() - return result diff --git a/models/icpr2020dfdc/environment.yml b/models/icpr2020dfdc/environment.yml deleted file mode 100644 index ca740bd791ffa840c362bac6a6af1d2355d33f33..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/environment.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: icpr2020 -channels: - - pytorch - - conda-forge - - defaults -dependencies: - - av=6.2.0 - - albumentations - - cudatoolkit - - ffmpeg - - jupyter - - numpy - - opencv=3.4.2 - - py-opencv=3.4.2 - - python=3.6.9 - - pip - - pytorch=1.4.0 - - torchvision - - tqdm - - pandas - - pip: - - tensorboardx==2.0 - - efficientnet-pytorch - - scikit-learn - diff --git a/models/icpr2020dfdc/extract_faces.py b/models/icpr2020dfdc/extract_faces.py deleted file mode 100644 index 3a699676cd7220280b3eb2a9926865afda7bcc94..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/extract_faces.py +++ /dev/null @@ -1,346 +0,0 @@ -""" -Extract faces - -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import argparse -import sys -import traceback -from concurrent.futures import ThreadPoolExecutor -from functools import partial -from pathlib import Path -from typing import Tuple, List - -import numpy as np -import pandas as pd -import torch -import torch.cuda -from PIL import Image -from tqdm import tqdm - -import blazeface -from blazeface import BlazeFace, VideoReader, FaceExtractor -from isplutils.utils import adapt_bb - - -def parse_args(argv): - parser = argparse.ArgumentParser() - parser.add_argument('--source', type=Path, help='Videos root directory', required=True) - parser.add_argument('--videodf', type=Path, help='Path to read the videos DataFrame', required=True) - parser.add_argument('--facesfolder', type=Path, help='Faces output root directory', required=True) - parser.add_argument('--facesdf', type=Path, help='Path to save the output DataFrame of faces', required=True) - parser.add_argument('--checkpoint', type=Path, help='Path to save the temporary per-video outputs', required=True) - - parser.add_argument('--fpv', type=int, default=32, help='Frames per video') - parser.add_argument('--device', type=torch.device, - default=torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'), - help='Device to use for face extraction') - parser.add_argument('--collateonly', help='Only perform collation of pre-existing results', action='store_true') - parser.add_argument('--noindex', help='Do not rebuild the index', action='store_false') - parser.add_argument('--batch', type=int, help='Batch size', default=16) - parser.add_argument('--threads', type=int, help='Number of threads', default=8) - parser.add_argument('--offset', type=int, help='Offset to start extraction', default=0) - parser.add_argument('--num', type=int, help='Number of videos to process', default=0) - parser.add_argument('--lazycheck', action='store_true', help='Lazy check of existing video indexes') - parser.add_argument('--deepcheck', action='store_true', help='Try to open every image') - - return parser.parse_args(argv) - - -def main(argv): - args = parse_args(argv) - - ## Parameters parsing - device: torch.device = args.device - source_dir: Path = args.source - facedestination_dir: Path = args.facesfolder - frames_per_video: int = args.fpv - videodataset_path: Path = args.videodf - facesdataset_path: Path = args.facesdf - collateonly: bool = args.collateonly - batch_size: int = args.batch - threads: int = args.threads - offset: int = args.offset - num: int = args.num - lazycheck: bool = args.lazycheck - deepcheck: bool = args.deepcheck - checkpoint_folder: Path = args.checkpoint - index_enable: bool = args.noindex - - ## Parameters - face_size = 512 - - print('Loading video DataFrame') - df_videos = pd.read_pickle(videodataset_path) - - if num > 0: - df_videos_process = df_videos.iloc[offset:offset + num] - else: - df_videos_process = df_videos.iloc[offset:] - - if not collateonly: - - ## Blazeface loading - print('Loading face extractor') - facedet = BlazeFace().to(device) - facedet.load_weights("blazeface/blazeface.pth") - facedet.load_anchors("blazeface/anchors.npy") - videoreader = VideoReader(verbose=False) - video_read_fn = lambda x: videoreader.read_frames(x, num_frames=frames_per_video) - face_extractor = FaceExtractor(video_read_fn, facedet) - - ## Face extraction - with ThreadPoolExecutor(threads) as p: - for batch_idx0 in tqdm(np.arange(start=0, stop=len(df_videos_process), step=batch_size), - desc='Extracting faces'): - tosave_list = list(p.map(partial(process_video, - source_dir=source_dir, - facedestination_dir=facedestination_dir, - checkpoint_folder=checkpoint_folder, - face_size=face_size, - face_extractor=face_extractor, - lazycheck=lazycheck, - deepcheck=deepcheck, - ), - df_videos_process.iloc[batch_idx0:batch_idx0 + batch_size].iterrows())) - - for tosave in tosave_list: - if tosave is not None: - if len(tosave[2]): - list(p.map(save_jpg, tosave[2])) - tosave[1].parent.mkdir(parents=True, exist_ok=True) - tosave[0].to_pickle(str(tosave[1])) - - if index_enable: - # Collect checkpoints - df_videos['nfaces'] = np.zeros(len(df_videos), np.uint8) - faces_dataset = [] - for idx, record in tqdm(df_videos.iterrows(), total=len(df_videos), desc='Collecting faces results'): - # Checkpoint - video_face_checkpoint_path = checkpoint_folder.joinpath(record['path']).with_suffix('.faces.pkl') - if video_face_checkpoint_path.exists(): - try: - df_video_faces = pd.read_pickle(str(video_face_checkpoint_path)) - # Fix same attribute issue - df_video_faces = df_video_faces.rename(columns={'subject': 'videosubject'}, errors='ignore') - nfaces = len( - np.unique(df_video_faces.index.map(lambda x: int(x.split('_subj')[1].split('.jpg')[0])))) - df_videos.loc[idx, 'nfaces'] = nfaces - faces_dataset.append(df_video_faces) - except Exception as e: - print('Error while reading: {}'.format(video_face_checkpoint_path)) - print(e) - video_face_checkpoint_path.unlink() - - if len(faces_dataset) == 0: - raise ValueError(f'No checkpoint found from face extraction. ' - f'Is the the source path {source_dir} correct for the videos in your dataframe?') - - # Save videos with updated faces - print('Saving videos DataFrame to {}'.format(videodataset_path)) - df_videos.to_pickle(str(videodataset_path)) - - if offset > 0: - if num > 0: - if facesdataset_path.is_dir(): - facesdataset_path = facesdataset_path.joinpath( - 'faces_df_from_video_{}_to_video_{}.pkl'.format(offset, num + offset)) - else: - facesdataset_path = facesdataset_path.parent.joinpath( - str(facesdataset_path.parts[-1]).split('.')[0] + '_from_video_{}_to_video_{}.pkl'.format(offset, - num + offset)) - else: - if facesdataset_path.is_dir(): - facesdataset_path = facesdataset_path.joinpath('faces_df_from_video_{}.pkl'.format(offset)) - else: - facesdataset_path = facesdataset_path.parent.joinpath( - str(facesdataset_path.parts[-1]).split('.')[0] + '_from_video_{}.pkl'.format(offset)) - elif num > 0: - if facesdataset_path.is_dir(): - facesdataset_path = facesdataset_path.joinpath( - 'faces_df_from_video_{}_to_video_{}.pkl'.format(0, num)) - else: - facesdataset_path = facesdataset_path.parent.joinpath( - str(facesdataset_path.parts[-1]).split('.')[0] + '_from_video_{}_to_video_{}.pkl'.format(0, num)) - else: - if facesdataset_path.is_dir(): - facesdataset_path = facesdataset_path.joinpath('faces_df.pkl') # just a check if the path is a dir - - # Creates directory (if doesn't exist) - facesdataset_path.parent.mkdir(parents=True, exist_ok=True) - print('Saving faces DataFrame to {}'.format(facesdataset_path)) - df_faces = pd.concat(faces_dataset, axis=0, ) - df_faces['video'] = df_faces['video'].astype('category') - for key in ['kp1x', 'kp1y', 'kp2x', 'kp2y', 'kp3x', - 'kp3y', 'kp4x', 'kp4y', 'kp5x', 'kp5y', 'kp6x', 'kp6y', 'left', - 'top', 'right', 'bottom', ]: - df_faces[key] = df_faces[key].astype(np.int16) - df_faces['videosubject'] = df_faces['videosubject'].astype(np.int8) - # Eventually remove duplicates - df_faces = df_faces.loc[~df_faces.index.duplicated(keep='first')] - fields_to_preserve_from_video = [i for i in - ['folder', 'subject', 'scene', 'cluster', 'nfaces', 'test'] if - i in df_videos] - df_faces = pd.merge(df_faces, df_videos[fields_to_preserve_from_video], left_on='video', - right_index=True) - df_faces.to_pickle(str(facesdataset_path)) - - print('Completed!') - - -def save_jpg(args: Tuple[Image.Image, Path or str]): - image, path = args - image.save(path, quality=95, subsampling='4:4:4') - - -def process_video(item: Tuple[pd.Index, pd.Series], - source_dir: Path, - facedestination_dir: Path, - checkpoint_folder: Path, - face_size: int, - face_extractor: FaceExtractor, - lazycheck: bool = False, - deepcheck: bool = False, - ) -> (pd.DataFrame, Path, List[Tuple[Image.Image, Path]]) or None: - # Instatiate Index and Series - idx, record = item - - # Checkpoint - video_faces_checkpoint_path = checkpoint_folder.joinpath(record['path']).with_suffix('.faces.pkl') - - if not lazycheck: - if video_faces_checkpoint_path.exists(): - try: - df_video_faces = pd.read_pickle(str(video_faces_checkpoint_path)) - for _, r in df_video_faces.iterrows(): - face_path = facedestination_dir.joinpath(r.name) - assert (face_path.exists()) - if deepcheck: - img = Image.open(face_path) - img_arr = np.asarray(img) - assert (img_arr.ndim == 3) - assert (np.prod(img_arr.shape) > 0) - except Exception as e: - print('Error while checking: {}'.format(video_faces_checkpoint_path)) - print(e) - video_faces_checkpoint_path.unlink() - - if not (video_faces_checkpoint_path.exists()): - - try: - - video_face_dict_list = [] - - # Load faces - current_video_path = source_dir.joinpath(record['path']) - if not current_video_path.exists(): - raise FileNotFoundError(f'Unable to find {current_video_path}.' - f'Are you sure that {source_dir} is the correct source directory for the video ' - f'you indexed in the dataframe?') - - frames = face_extractor.process_video(current_video_path) - - if len(frames) == 0: - return - - face_extractor.keep_only_best_face(frames) - for frame_idx, frame in enumerate(frames): - frames[frame_idx]['subjects'] = [0] * len(frames[frame_idx]['detections']) - - # Extract and save faces, bounding boxes, keypoints - images_to_save: List[Tuple[Image.Image, Path]] = [] - for frame_idx, frame in enumerate(frames): - if len(frames[frame_idx]['detections']): - fullframe = Image.fromarray(frames[frame_idx]['frame']) - - # Preserve the only found face even if not a good one, otherwise preserve only clusters > -1 - subjects = np.unique(frames[frame_idx]['subjects']) - if len(subjects) > 1: - subjects = np.asarray([s for s in subjects if s > -1]) - - for face_idx, _ in enumerate(frame['faces']): - subj_id = frames[frame_idx]['subjects'][face_idx] - if subj_id in subjects: # Exclude outliers if other faces detected - face_path = facedestination_dir.joinpath(record['path'], 'fr{:03d}_subj{:1d}.jpg'.format( - frames[frame_idx]['frame_idx'], subj_id)) - - face_dict = {'facepath': str(face_path.relative_to(facedestination_dir)), 'video': idx, - 'label': record['label'], 'videosubject': subj_id, - 'original': record['original']} - # add attibutes for ff++ - if 'class' in record.keys(): - face_dict.update({'class': record['class']}) - if 'source' in record.keys(): - face_dict.update({'source': record['source']}) - if 'quality' in record.keys(): - face_dict.update({'quality': record['quality']}) - - for field_idx, key in enumerate(blazeface.BlazeFace.detection_keys): - face_dict[key] = frames[frame_idx]['detections'][face_idx][field_idx] - - cropping_bb = adapt_bb(frame_height=fullframe.height, - frame_width=fullframe.width, - bb_height=face_size, - bb_width=face_size, - left=face_dict['xmin'], - top=face_dict['ymin'], - right=face_dict['xmax'], - bottom=face_dict['ymax']) - face = fullframe.crop(cropping_bb) - - for key in blazeface.BlazeFace.detection_keys: - if (key[0] == 'k' and key[-1] == 'x') or (key[0] == 'x'): - face_dict[key] -= cropping_bb[0] - elif (key[0] == 'k' and key[-1] == 'y') or (key[0] == 'y'): - face_dict[key] -= cropping_bb[1] - - face_dict['left'] = face_dict.pop('xmin') - face_dict['top'] = face_dict.pop('ymin') - face_dict['right'] = face_dict.pop('xmax') - face_dict['bottom'] = face_dict.pop('ymax') - - face_path.parent.mkdir(parents=True, exist_ok=True) - images_to_save.append((face, face_path)) - - video_face_dict_list.append(face_dict) - - if len(video_face_dict_list) > 0: - - df_video_faces = pd.DataFrame(video_face_dict_list) - df_video_faces.index = df_video_faces['facepath'] - del df_video_faces['facepath'] - - # type conversions - for key in ['kp1x', 'kp1y', 'kp2x', 'kp2y', 'kp3x', 'kp3y', - 'kp4x', 'kp4y', 'kp5x', 'kp5y', 'kp6x', 'kp6y', 'left', 'top', - 'right', 'bottom']: - df_video_faces[key] = df_video_faces[key].astype(np.int16) - df_video_faces['conf'] = df_video_faces['conf'].astype(np.float32) - df_video_faces['video'] = df_video_faces['video'].astype('category') - - video_faces_checkpoint_path.parent.mkdir(parents=True, exist_ok=True) - - else: - print('No faces extracted for video {}'.format(record['path'])) - df_video_faces = pd.DataFrame() - - return df_video_faces, video_faces_checkpoint_path, images_to_save - - except Exception as e: - print('Error while processing: {}'.format(record['path'])) - print("-" * 60) - traceback.print_exc(file=sys.stdout, limit=5) - print("-" * 60) - return - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/models/icpr2020dfdc/index_celebdf.py b/models/icpr2020dfdc/index_celebdf.py deleted file mode 100644 index 68b50c6693b6c6438014f93c8ba32e44387d22f1..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/index_celebdf.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -Index Celeb-DF v2 -Image and Sound Processing Lab - Politecnico di Milano -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import argparse -from multiprocessing import Pool -from pathlib import Path - -import numpy as np -import pandas as pd - -from isplutils.utils import extract_meta_av, extract_meta_cv - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--source', type=Path, help='Source dir', - required=True) - parser.add_argument('--videodataset', type=Path, default='data/celebdf_videos.pkl', - help='Path to save the videos DataFrame') - - args = parser.parse_args() - - ## Parameters parsing - source_dir: Path = args.source - videodataset_path: Path = args.videodataset - - # Create ouput folder (if doesn't exist) - videodataset_path.parent.mkdir(parents=True, exist_ok=True) - - ## DataFrame - if videodataset_path.exists(): - print('Loading video DataFrame') - df_videos = pd.read_pickle(videodataset_path) - else: - print('Creating video DataFrame') - - split_file = Path(source_dir).joinpath('List_of_testing_videos.txt') - if not split_file.exists(): - raise FileNotFoundError('Unable to find "List_of_testing_videos.txt" in {}'.format(source_dir)) - test_videos_df = pd.read_csv(split_file, delimiter=' ', header=0, index_col=1) - - ff_videos = Path(source_dir).rglob('*.mp4') - df_videos = pd.DataFrame( - {'path': [f.relative_to(source_dir) for f in ff_videos]}) - - df_videos['height'] = df_videos['width'] = df_videos['frames'] = np.zeros(len(df_videos), dtype=np.uint16) - with Pool() as p: - meta = p.map(extract_meta_av, df_videos['path'].map(lambda x: str(source_dir.joinpath(x)))) - meta = np.stack(meta) - df_videos.loc[:, ['height', 'width', 'frames']] = meta - - # Fix for videos that av cannot decode properly - for idx, record in df_videos[df_videos['frames'] == 0].iterrows(): - meta = extract_meta_cv(str(source_dir.joinpath(record['path']))) - df_videos.loc[idx, ['height', 'width', 'frames']] = meta - - df_videos['class'] = df_videos['path'].map(lambda x: x.parts[0]).astype('category') - df_videos['label'] = df_videos['class'].map( - lambda x: True if x == 'Celeb-synthesis' else False) # True is FAKE, False is REAL - df_videos['name'] = df_videos['path'].map(lambda x: x.with_suffix('').name) - - df_videos['original'] = -1 * np.ones(len(df_videos), dtype=np.int16) - df_videos.loc[(df_videos['label'] == True), 'original'] = \ - df_videos[(df_videos['label'] == True)]['name'].map( - lambda x: df_videos.index[ - np.flatnonzero(df_videos['name'] == '_'.join([x.split('_')[0], x.split('_')[2]]))[0]] - ) - - df_videos['test'] = df_videos['path'].map(str).isin(test_videos_df.index) - - print('Saving video DataFrame to {}'.format(videodataset_path)) - df_videos.to_pickle(str(videodataset_path)) - - print('Real videos: {:d}'.format(sum(df_videos['label'] == 0))) - print('Fake videos: {:d}'.format(sum(df_videos['label'] == 1))) - - -if __name__ == '__main__': - main() diff --git a/models/icpr2020dfdc/index_dfdc.py b/models/icpr2020dfdc/index_dfdc.py deleted file mode 100644 index ddad62de137826ac15fae25c9dc88dfe55514cd0..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/index_dfdc.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -Index the official Kaggle training dataset and prepares a train and validation set based on folders - -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import sys -import argparse -from multiprocessing import Pool -from pathlib import Path - -import numpy as np -import pandas as pd -from tqdm import tqdm - -from isplutils.utils import extract_meta_av - - -def parse_args(argv): - parser = argparse.ArgumentParser() - parser.add_argument('--source', type=Path, help='Source dir', required=True) - parser.add_argument('--videodataset', type=Path, default='data/dfdc_videos.pkl', - help='Path to save the videos DataFrame') - parser.add_argument('--batch', type=int, help='Batch size', default=64) - - return parser.parse_args(argv) - - -def main(argv): - ## Parameters parsing - args = parse_args(argv) - source_dir: Path = args.source - videodataset_path: Path = args.videodataset - batch_size: int = args.batch - - ## DataFrame - if videodataset_path.exists(): - print('Loading video DataFrame') - df_videos = pd.read_pickle(videodataset_path) - else: - print('Creating video DataFrame') - - # Create ouptut folder - videodataset_path.parent.mkdir(parents=True, exist_ok=True) - - # Index - df_train_list = list() - for idx, json_path in enumerate(tqdm(sorted(source_dir.rglob('metadata.json')), desc='Indexing')): - df_tmp = pd.read_json(json_path, orient='index') - df_tmp['path'] = df_tmp.index.map( - lambda x: str(json_path.parent.relative_to(source_dir).joinpath(x))) - df_tmp['folder'] = int(str(json_path.parts[-2]).split('_')[-1]) - df_train_list.append(df_tmp) - df_videos = pd.concat(df_train_list, axis=0, verify_integrity=True) - - # Save space - del df_videos['split'] - df_videos['label'] = df_videos['label'] == 'FAKE' - df_videos['original'] = df_videos['original'].astype('category') - df_videos['folder'] = df_videos['folder'].astype(np.uint8) - - # Collect metadata - paths_arr = np.asarray(df_videos.path.map(lambda x: str(source_dir.joinpath(x)))) - height_list = [] - width_list = [] - frames_list = [] - with Pool() as pool: - for batch_idx0 in tqdm(np.arange(start=0, stop=len(df_videos), step=batch_size), desc='Metadata'): - batch_res = pool.map(extract_meta_av, paths_arr[batch_idx0:batch_idx0 + batch_size]) - for res in batch_res: - height_list.append(res[0]) - width_list.append(res[1]) - frames_list.append(res[2]) - - df_videos['height'] = np.asarray(height_list, dtype=np.uint16) - df_videos['width'] = np.asarray(width_list, dtype=np.uint16) - df_videos['frames'] = np.asarray(frames_list, dtype=np.uint16) - - print('Saving video DataFrame to {}'.format(videodataset_path)) - df_videos.to_pickle(str(videodataset_path)) - - print('Real videos: {:d}'.format(sum(df_videos['label'] == 0))) - print('Fake videos: {:d}'.format(sum(df_videos['label'] == 1))) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/models/icpr2020dfdc/index_ffpp.py b/models/icpr2020dfdc/index_ffpp.py deleted file mode 100644 index cbb25d72a0525e0159642d1ef063826ace72181e..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/index_ffpp.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Index FaceForensics++ - -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import argparse -import sys -from multiprocessing import Pool -from pathlib import Path - -import numpy as np -import pandas as pd - -from isplutils.utils import extract_meta_av, extract_meta_cv - - -def parse_args(argv): - parser = argparse.ArgumentParser() - parser.add_argument('--source', type=Path, help='Source dir', - default='dataset/ffpp/faceforensics') - parser.add_argument('--videodataset', type=Path, default='data/ffpp_videos.pkl', - help='Path to save the videos DataFrame') - - return parser.parse_args(argv) - - -def main(argv): - ## Parameters parsing - args = parse_args(argv) - source_dir: Path = args.source - videodataset_path: Path = args.videodataset - - # Create ouput folder (if doesn't exist) - videodataset_path.parent.mkdir(parents=True, exist_ok=True) - - ## DataFrame - if videodataset_path.exists(): - print('Loading video DataFrame') - df_videos = pd.read_pickle(videodataset_path) - else: - print('Creating video DataFrame') - - ff_videos = Path(source_dir).rglob('*.mp4') - df_videos = pd.DataFrame( - {'path': [f.relative_to(source_dir) for f in ff_videos if 'mask' not in str(f) and 'raw' not in str(f)]}) - - df_videos['height'] = df_videos['width'] = df_videos['frames'] = np.zeros(len(df_videos), dtype=np.uint16) - with Pool() as p: - meta = p.map(extract_meta_av, df_videos['path'].map(lambda x: str(source_dir.joinpath(x)))) - meta = np.stack(meta) - df_videos.loc[:, ['height', 'width', 'frames']] = meta - - # Fix for videos that av cannot decode properly - for idx, record in df_videos[df_videos['frames'] == 0].iterrows(): - meta = extract_meta_cv(str(source_dir.joinpath(record['path']))) - df_videos.loc[idx, ['height', 'width', 'frames']] = meta - - df_videos['class'] = df_videos['path'].map(lambda x: x.parts[0]).astype('category') - df_videos['label'] = df_videos['class'].map( - lambda x: True if x == 'manipulated_sequences' else False) # True is FAKE, False is REAL - df_videos['source'] = df_videos['path'].map(lambda x: x.parts[1]).astype('category') - df_videos['quality'] = df_videos['path'].map(lambda x: x.parts[2]).astype('category') - df_videos['name'] = df_videos['path'].map(lambda x: x.with_suffix('').parts[-1]) - - df_videos['original'] = -1 * np.ones(len(df_videos), dtype=np.int16) - df_videos.loc[(df_videos['label'] == True) & (df_videos['source'] != 'DeepFakeDetection'), 'original'] = \ - df_videos[(df_videos['label'] == True) & (df_videos['source'] != 'DeepFakeDetection')]['name'].map( - lambda x: df_videos.index[np.flatnonzero(df_videos['name'] == x.split('_')[0])[0]] - ) - df_videos.loc[(df_videos['label'] == True) & (df_videos['source'] == 'DeepFakeDetection'), 'original'] = \ - df_videos[(df_videos['label'] == True) & (df_videos['source'] == 'DeepFakeDetection')]['name'].map( - lambda x: df_videos.index[ - np.flatnonzero(df_videos['name'] == x.split('_')[0] + '__' + x.split('__')[1])[0]] - ) - - print('Saving video DataFrame to {}'.format(videodataset_path)) - df_videos.to_pickle(str(videodataset_path)) - - print('Real videos: {:d}'.format(sum(df_videos['label'] == 0))) - print('Fake videos: {:d}'.format(sum(df_videos['label'] == 1))) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/models/icpr2020dfdc/isplutils/__init__.py b/models/icpr2020dfdc/isplutils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/models/icpr2020dfdc/isplutils/data.py b/models/icpr2020dfdc/isplutils/data.py deleted file mode 100644 index 86b788ebec86c4298aaf4d89e5fdf7f8efe938c5..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/isplutils/data.py +++ /dev/null @@ -1,263 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import os -from pathlib import Path -from typing import List - -import albumentations as A -import numpy as np -import pandas as pd -import torch -from PIL import Image -from albumentations.pytorch import ToTensorV2 -from torch.utils.data import Dataset, IterableDataset - -from .utils import extract_bb - - -def load_face(record: pd.Series, root: str, size: int, scale: str, transformer: A.BasicTransform) -> torch.Tensor: - path = os.path.join(str(root), str(record.name)) - autocache = size < 256 or scale == 'tight' - if scale in ['crop', 'scale', ]: - cached_path = str(Path(root).joinpath('autocache', scale, str(size), str(record.name)).with_suffix('.jpg')) - else: - # when self.scale == 'tight' the extracted face is not dependent on size - cached_path = str(Path(root).joinpath('autocache', scale, str(record.name)).with_suffix('.jpg')) - - face = np.zeros((size, size, 3), dtype=np.uint8) - if os.path.exists(cached_path): - try: - face = Image.open(cached_path) - face = np.array(face) - if len(face.shape) != 3: - raise RuntimeError('Incorrect format: {}'.format(path)) - except KeyboardInterrupt as e: - # We want keybord interrupts to be propagated - raise e - except (OSError, IOError) as e: - print('Deleting corrupted cache file: {}'.format(cached_path)) - print(e) - os.unlink(cached_path) - face = np.zeros((size, size, 3), dtype=np.uint8) - - if not os.path.exists(cached_path): - try: - frame = Image.open(path) - bb = record['left'], record['top'], record['right'], record['bottom'] - face = extract_bb(frame, bb=bb, size=size, scale=scale) - - if autocache: - os.makedirs(os.path.dirname(cached_path), exist_ok=True) - face.save(cached_path, quality=95, subsampling='4:4:4') - - face = np.array(face) - if len(face.shape) != 3: - raise RuntimeError('Incorrect format: {}'.format(path)) - except KeyboardInterrupt as e: - # We want keybord interrupts to be propagated - raise e - except (OSError, IOError) as e: - print('Error while reading: {}'.format(path)) - print(e) - face = np.zeros((size, size, 3), dtype=np.uint8) - - face = transformer(image=face)['image'] - - return face - - -class FrameFaceIterableDataset(IterableDataset): - - def __init__(self, - roots: List[str], - dfs: List[pd.DataFrame], - size: int, scale: str, - num_samples: int = -1, - transformer: A.BasicTransform = ToTensorV2(), - output_index: bool = False, - labels_map: dict = None, - seed: int = None): - """ - - :param roots: List of root folders for frames cache - :param dfs: List of DataFrames of cached frames with 'bb' column as array of 4 elements (left,top,right,bottom) - and 'label' column - :param size: face size - :param num_samples: - :param scale: Rescale the face to the given size, preserving the aspect ratio. - If false crop around center to the given size - :param transformer: - :param output_index: enable output of df_frames index - :param labels_map: map from 'REAL' and 'FAKE' to actual labels - """ - - self.dfs = dfs - self.size = int(size) - - self.seed0 = int(seed) if seed is not None else np.random.choice(2 ** 32) - - # adapt indices - dfs_adapted = [df.copy() for df in self.dfs] - for df_idx, df in enumerate(dfs_adapted): - mi = pd.MultiIndex.from_tuples([(df_idx, key) for key in df.index], names=['df_idx', 'df_key']) - df.index = mi - # Concat - self.df = pd.concat(dfs_adapted, axis=0, join='inner') - - self.df_real = self.df[self.df['label'] == 0] - self.df_fake = self.df[self.df['label'] == 1] - - self.longer_set = 'real' if len(self.df_real) > len(self.df_fake) else 'fake' - self.num_samples = max(len(self.df_real), len(self.df_fake)) * 2 - self.num_samples = min(self.num_samples, num_samples) if num_samples > 0 else self.num_samples - - self.output_idx = bool(output_index) - - self.scale = str(scale) - self.roots = [str(r) for r in roots] - self.transformer = transformer - - self.labels_map = labels_map - if self.labels_map is None: - self.labels_map = {False: np.array([0., ]), True: np.array([1., ])} - else: - self.labels_map = dict(self.labels_map) - - def _get_face(self, item: pd.Index) -> (torch.Tensor, torch.Tensor) or (torch.Tensor, torch.Tensor, str): - - record = self.dfs[item[0]].loc[item[1]] - face = load_face(record=record, - root=self.roots[item[0]], - size=self.size, - scale=self.scale, - transformer=self.transformer) - - label = self.labels_map[record.label] - if self.output_idx: - return face, label, record.name - else: - return face, label - - def __len__(self): - return self.num_samples - - def __iter__(self): - - random_fake_idxs, random_real_idxs = get_iterative_real_fake_idxs( - df_real=self.df_real, - df_fake=self.df_fake, - num_samples=self.num_samples, - seed0=self.seed0 - ) - - while len(random_fake_idxs) >= 1 and len(random_real_idxs) >= 1: - yield self._get_face(random_fake_idxs.pop()) - yield self._get_face(random_real_idxs.pop()) - - -def get_iterative_real_fake_idxs(df_real: pd.DataFrame, df_fake: pd.DataFrame, - num_samples: int, seed0: int): - longer_set = 'real' if len(df_real) > len(df_fake) else 'fake' - worker_info = torch.utils.data.get_worker_info() - if worker_info is None: - seed = seed0 - np.random.seed(seed) - worker_num_couple_samples = num_samples // 2 - fake_idxs_portion = np.random.choice(df_fake.index, worker_num_couple_samples, - replace=longer_set == 'real') - real_idxs_portion = np.random.choice(df_real.index, worker_num_couple_samples, - replace=longer_set == 'fake') - else: - worker_id = worker_info.id - seed = seed0 + worker_id - np.random.seed(seed) - worker_num_couple_samples = (num_samples // 2) // worker_info.num_workers - if longer_set == 'fake': - fake_idxs_portion = df_fake.index[ - worker_id * worker_num_couple_samples:(worker_id + 1) * worker_num_couple_samples] - real_idxs_portion = np.random.choice(df_real.index, worker_num_couple_samples, replace=True) - else: - real_idxs_portion = df_real.index[ - worker_id * worker_num_couple_samples:(worker_id + 1) * worker_num_couple_samples] - fake_idxs_portion = np.random.choice(df_fake.index, worker_num_couple_samples, - replace=True) - random_fake_idxs = list(np.random.permutation(fake_idxs_portion)) - random_real_idxs = list(np.random.permutation(real_idxs_portion)) - - assert (len(random_fake_idxs) == len(random_real_idxs)) - - return random_fake_idxs, random_real_idxs - - -class FrameFaceDatasetTest(Dataset): - - def __init__(self, root: str, df: pd.DataFrame, - size: int, scale: str, - transformer: A.BasicTransform = ToTensorV2(), - labels_map: dict = None, - aug_transformers: List[A.BasicTransform] = None): - """ - - :param root: root folder for frames cache - :param df: DataFrame of cached frames with 'bb' column as array of 4 elements (left,top,right,bottom) - and 'label' column - :param size: face size - :param num_samples: - :param scale: Rescale the face to the given size, preserving the aspect ratio. - If false crop around center to the given size - :param transformer: - :param labels_map: dcit to map df labels - :param aug_transformers: if not None, creates multiple copies of the same sample according to the provided augmentations - """ - - self.df = df - self.size = int(size) - - self.scale = str(scale) - self.root = str(root) - self.transformer = transformer - self.aug_transformers = aug_transformers - - self.labels_map = labels_map - if self.labels_map is None: - self.labels_map = {False: np.array([0., ]), True: np.array([1., ])} - else: - self.labels_map = dict(self.labels_map) - - def _get_face(self, item: pd.Index) -> (torch.Tensor, torch.Tensor) or (torch.Tensor, torch.Tensor, str): - record = self.df.loc[item] - label = self.labels_map[record.label] - if self.aug_transformers is None: - face = load_face(record=record, - root=self.root, - size=self.size, - scale=self.scale, - transformer=self.transformer) - return face, label - else: - faces = [] - for aug_transf in self.aug_transformers: - faces.append( - load_face(record=record, - root=self.root, - size=self.size, - scale=self.scale, - transformer=A.Compose([aug_transf, self.transformer]) - )) - faces = torch.stack(faces) - return faces, label - - def __len__(self): - return len(self.df) - - def __getitem__(self, item): - return self._get_face(self.df.index[item]) diff --git a/models/icpr2020dfdc/isplutils/data_siamese.py b/models/icpr2020dfdc/isplutils/data_siamese.py deleted file mode 100644 index d883291bc2bbbb898330f761e0a95b399364a7cc..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/isplutils/data_siamese.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -from typing import List - -import albumentations as A -import pandas as pd -from albumentations.pytorch import ToTensorV2 - -from .data import FrameFaceIterableDataset, get_iterative_real_fake_idxs - - -class FrameFaceTripletIterableDataset(FrameFaceIterableDataset): - - def __init__(self, - roots: List[str], - dfs: List[pd.DataFrame], - size: int, - scale: str, - num_triplets: int = -1, - transformer: A.BasicTransform = ToTensorV2(), - seed: int = None): - """ - - :param roots: List of root folders for frames cache - :param dfs: List of DataFrames of cached frames with 'bb' column as array of 4 elements (left,top,right,bottom) - and 'label' column - :param size: face size - :param num_triplets: number of samples for the dataset - :param idxs: sampling indexes triplets (each element is a key for anchor, positive, negative) - :param scale: Rescale the face to the given size, preserving the aspect ratio. - If false crop around center to the given size - :param transformer: - :param seed: - """ - super(FrameFaceTripletIterableDataset, self).__init__( - roots=roots, - dfs=dfs, - size=size, - scale=scale, - num_samples=num_triplets * 3, - transformer=transformer, - seed=seed - ) - - self.num_triplet_couples = self.num_samples // 6 - self.num_triplets = self.num_triplet_couples * 2 - self.num_samples = self.num_triplets * 3 - - def __len__(self): - return self.num_triplets - - def __iter__(self): - random_fake_idxs, random_real_idxs = get_iterative_real_fake_idxs( - df_real=self.df_real, - df_fake=self.df_fake, - num_samples=self.num_samples, - seed0=self.seed0 - ) - - while len(random_fake_idxs) >= 3 and len(random_real_idxs) >= 3: - a = self._get_face(random_fake_idxs.pop())[0] - p = self._get_face(random_fake_idxs.pop())[0] - n = self._get_face(random_real_idxs.pop())[0] - yield a, p, n - - a = self._get_face(random_real_idxs.pop())[0] - p = self._get_face(random_real_idxs.pop())[0] - n = self._get_face(random_fake_idxs.pop())[0] - yield a, p, n diff --git a/models/icpr2020dfdc/isplutils/split.py b/models/icpr2020dfdc/isplutils/split.py deleted file mode 100644 index b9eb8139cdc7da92e72d1dd5e83db70175630898..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/isplutils/split.py +++ /dev/null @@ -1,135 +0,0 @@ -from typing import List, Dict, Tuple -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import numpy as np -import pandas as pd - -available_datasets = [ - 'dfdc-35-5-10', - 'ff-c23-720-140-140', - 'ff-c23-720-140-140-5fpv', - 'ff-c23-720-140-140-10fpv', - 'ff-c23-720-140-140-15fpv', - 'ff-c23-720-140-140-20fpv', - 'ff-c23-720-140-140-25fpv', - 'celebdf', # just for convenience, not used in the original paper -] - - -def load_df(dfdc_df_path: str, ffpp_df_path: str, dfdc_faces_dir: str, ffpp_faces_dir: str, dataset: str) -> (pd.DataFrame, str): - if dataset.startswith('dfdc'): - df = pd.read_pickle(dfdc_df_path) - root = dfdc_faces_dir - elif dataset.startswith('ff-'): - df = pd.read_pickle(ffpp_df_path) - root = ffpp_faces_dir - else: - raise NotImplementedError('Unknown dataset: {}'.format(dataset)) - return df, root - - -def get_split_df(df: pd.DataFrame, dataset: str, split: str) -> pd.DataFrame: - if dataset == 'dfdc-35-5-10': - if split == 'train': - split_df = df[df['folder'].isin(range(35))] - elif split == 'val': - split_df = df[df['folder'].isin(range(35, 40))] - elif split == 'test': - split_df = df[df['folder'].isin(range(40, 50))] - else: - raise NotImplementedError('Unknown split: {}'.format(split)) - elif dataset.startswith('ff-c23-720-140-140'): - # Save random state - st0 = np.random.get_state() - # Set seed for this selection only - np.random.seed(41) - # Split on original videos - crf = dataset.split('-')[1] - random_youtube_videos = np.random.permutation( - df[(df['source'] == 'youtube') & (df['quality'] == crf)]['video'].unique()) - train_orig = random_youtube_videos[:720] - val_orig = random_youtube_videos[720:720 + 140] - test_orig = random_youtube_videos[720 + 140:] - if split == 'train': - split_df = pd.concat((df[df['original'].isin(train_orig)], df[df['video'].isin(train_orig)]), axis=0) - elif split == 'val': - split_df = pd.concat((df[df['original'].isin(val_orig)], df[df['video'].isin(val_orig)]), axis=0) - elif split == 'test': - split_df = pd.concat((df[df['original'].isin(test_orig)], df[df['video'].isin(test_orig)]), axis=0) - else: - raise NotImplementedError('Unknown split: {}'.format(split)) - - if dataset.endswith('fpv'): - fpv = int(dataset.rsplit('-', 1)[1][:-3]) - idxs = [] - for video in split_df['video'].unique(): - idxs.append(np.random.choice(split_df[split_df['video'] == video].index, fpv, replace=False)) - idxs = np.concatenate(idxs) - split_df = split_df.loc[idxs] - # Restore random state - np.random.set_state(st0) - elif dataset == 'celebdf': - - seed = 41 - num_real_train = 600 - - # Save random state - st0 = np.random.get_state() - # Set seed for this selection only - np.random.seed(seed) - # Split on original videos - random_train_val_real_videos = np.random.permutation( - df[(df['label'] == False) & (df['test'] == False)]['video'].unique()) - train_orig = random_train_val_real_videos[:num_real_train] - val_orig = random_train_val_real_videos[num_real_train:] - if split == 'train': - split_df = pd.concat((df[df['original'].isin(train_orig)], df[df['video'].isin(train_orig)]), axis=0) - elif split == 'val': - split_df = pd.concat((df[df['original'].isin(val_orig)], df[df['video'].isin(val_orig)]), axis=0) - elif split == 'test': - split_df = df[df['test'] == True] - else: - raise NotImplementedError('Unknown split: {}'.format(split)) - # Restore random state - np.random.set_state(st0) - else: - raise NotImplementedError('Unknown dataset: {}'.format(dataset)) - return split_df - - -def make_splits(dfdc_df: str, ffpp_df: str, dfdc_dir: str, ffpp_dir: str, dbs: Dict[str, List[str]]) -> Dict[str, Dict[str, Tuple[pd.DataFrame, str]]]: - """ - Make split and return Dataframe and root - :param - dfdc_df: str, path to the DataFrame containing info on the faces extracted from the DFDC dataset with extract_faces.py - ffpp_df: str, path to the DataFrame containing info on the faces extracted from the FF++ dataset with extract_faces.py - dfdc_dir: str, path to the directory containing the faces extracted from the DFDC dataset with extract_faces.py - ffpp_dir: str, path to the directory containing the faces extracted from the FF++ dataset with extract_faces.py - dbs: {split_name:[split_dataset1,split_dataset2,...]} - Example: - {'train':['dfdc-35-5-15',],'val':['dfdc-35-5-15',]} - :return: split_dict: dictonary containing {split_name: ['train', 'val'], splitdb: List(pandas.DataFrame, str)} - Example: - {'train, 'dfdc-35-5-15': (dfdc_train_df, 'path/to/dir/of/DFDC/faces')} - """ - split_dict = {} - full_dfs = {} - for split_name, split_dbs in dbs.items(): - split_dict[split_name] = dict() - for split_db in split_dbs: - if split_db not in full_dfs: - full_dfs[split_db] = load_df(dfdc_df, ffpp_df, dfdc_dir, ffpp_dir, split_db) - full_df, root = full_dfs[split_db] - split_df = get_split_df(df=full_df, dataset=split_db, split=split_name) - split_dict[split_name][split_db] = (split_df, root) - - return split_dict diff --git a/models/icpr2020dfdc/isplutils/utils.py b/models/icpr2020dfdc/isplutils/utils.py deleted file mode 100644 index 97c7c9bc2ab5bfe3c6046eb42c7a6734b384d329..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/isplutils/utils.py +++ /dev/null @@ -1,247 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -from pprint import pprint -from typing import Iterable, List - -import albumentations as A -import cv2 -import numpy as np -import scipy -import torch -from PIL import Image -from albumentations.pytorch import ToTensorV2 -from matplotlib import pyplot as plt -from torch import nn as nn -from torchvision import transforms - - -def extract_meta_av(path: str) -> (int, int, int): - """ - Extract video height, width and number of frames to index the files - :param path: - :return: - """ - import av - try: - video = av.open(path) - video_stream = video.streams.video[0] - return video_stream.height, video_stream.width, video_stream.frames - except av.AVError as e: - print('Error while reading file: {}'.format(path)) - print(e) - return 0, 0, 0 - except IndexError as e: - print('Error while processing file: {}'.format(path)) - print(e) - return 0, 0, 0 - - -def extract_meta_cv(path: str) -> (int, int, int): - """ - Extract video height, width and number of frames to index the files - :param path: - :return: - """ - try: - vid = cv2.VideoCapture(path) - num_frames = int(vid.get(cv2.CAP_PROP_FRAME_COUNT)) - height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT)) - width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH)) - return height, width, num_frames - except Exception as e: - print('Error while reading file: {}'.format(path)) - print(e) - return 0, 0, 0 - - -def adapt_bb(frame_height: int, frame_width: int, bb_height: int, bb_width: int, left: int, top: int, right: int, - bottom: int) -> ( - int, int, int, int): - x_ctr = (left + right) // 2 - y_ctr = (bottom + top) // 2 - new_top = max(y_ctr - bb_height // 2, 0) - new_bottom = min(new_top + bb_height, frame_height) - new_left = max(x_ctr - bb_width // 2, 0) - new_right = min(new_left + bb_width, frame_width) - return new_left, new_top, new_right, new_bottom - - -def extract_bb(frame: Image.Image, bb: Iterable, scale: str, size: int) -> Image.Image: - """ - Extract a face from a frame according to the given bounding box and scale policy - :param frame: Entire frame - :param bb: Bounding box (left,top,right,bottom) in the reference system of the frame - :param scale: "scale" to crop a square with size equal to the maximum between height and width of the face, then scale to size - "crop" to crop a fixed square around face center, - "tight" to crop face exactly at the bounding box with no scaling - :param size: size of the face - :return: - """ - left, top, right, bottom = bb - if scale == "scale": - bb_width = int(right) - int(left) - bb_height = int(bottom) - int(top) - bb_to_desired_ratio = min(size / bb_height, size / bb_width) if (bb_width > 0 and bb_height > 0) else 1. - bb_width = int(size / bb_to_desired_ratio) - bb_height = int(size / bb_to_desired_ratio) - left, top, right, bottom = adapt_bb(frame.height, frame.width, bb_height, bb_width, left, top, right, - bottom) - face = frame.crop((left, top, right, bottom)).resize((size, size), Image.BILINEAR) - elif scale == "crop": - # Find the center of the bounding box and cut an area around it of height x width - left, top, right, bottom = adapt_bb(frame.height, frame.width, size, size, left, top, right, - bottom) - face = frame.crop((left, top, right, bottom)) - elif scale == "tight": - left, top, right, bottom = adapt_bb(frame.height, frame.width, bottom - top, right - left, left, top, right, - bottom) - face = frame.crop((left, top, right, bottom)) - else: - raise ValueError('Unknown scale value: {}'.format(scale)) - - return face - - -def showimage(img_tensor: torch.Tensor): - topil = transforms.Compose([ - transforms.Normalize(mean=[0, 0, 0, ], std=[1 / 0.229, 1 / 0.224, 1 / 0.225]), - transforms.Normalize(mean=[-0.485, -0.456, -0.406], std=[1, 1, 1]), - transforms.ToPILImage() - ]) - plt.figure() - plt.imshow(topil(img_tensor)) - plt.show() - - -def make_train_tag(net_class: nn.Module, - face_policy: str, - patch_size: int, - traindb: List[str], - seed: int, - suffix: str, - debug: bool, - ): - # Training parameters and tag - tag_params = dict(net=net_class.__name__, - traindb='-'.join(traindb), - face=face_policy, - size=patch_size, - seed=seed - ) - print('Parameters') - pprint(tag_params) - tag = 'debug_' if debug else '' - tag += '_'.join(['-'.join([key, str(tag_params[key])]) for key in tag_params]) - if suffix is not None: - tag += '_' + suffix - print('Tag: {:s}'.format(tag)) - return tag - - -def get_transformer(face_policy: str, patch_size: int, net_normalizer: transforms.Normalize, train: bool): - # Transformers and traindb - if face_policy == 'scale': - # The loader crops the face isotropically then scales to a square of size patch_size_load - loading_transformations = [ - A.PadIfNeeded(min_height=patch_size, min_width=patch_size, - border_mode=cv2.BORDER_CONSTANT, value=0,always_apply=True), - A.Resize(height=patch_size,width=patch_size,always_apply=True), - ] - if train: - downsample_train_transformations = [ - A.Downscale(scale_max=0.5, scale_min=0.5, p=0.5), # replaces scaled dataset - ] - else: - downsample_train_transformations = [] - elif face_policy == 'tight': - # The loader crops the face tightly without any scaling - loading_transformations = [ - A.LongestMaxSize(max_size=patch_size, always_apply=True), - A.PadIfNeeded(min_height=patch_size, min_width=patch_size, - border_mode=cv2.BORDER_CONSTANT, value=0,always_apply=True), - ] - if train: - downsample_train_transformations = [ - A.Downscale(scale_max=0.5, scale_min=0.5, p=0.5), # replaces scaled dataset - ] - else: - downsample_train_transformations = [] - else: - raise ValueError('Unknown value for face_policy: {}'.format(face_policy)) - - if train: - aug_transformations = [ - A.Compose([ - A.HorizontalFlip(), - A.OneOf([ - A.RandomBrightnessContrast(), - A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=30, val_shift_limit=20), - ]), - A.OneOf([ - A.ISONoise(), - A.IAAAdditiveGaussianNoise(scale=(0.01 * 255, 0.03 * 255)), - ]), - A.Downscale(scale_min=0.7, scale_max=0.9, interpolation=cv2.INTER_LINEAR), - A.ImageCompression(quality_lower=50, quality_upper=99), - ], ) - ] - else: - aug_transformations = [] - - # Common final transformations - final_transformations = [ - A.Normalize(mean=net_normalizer.mean, std=net_normalizer.std, ), - ToTensorV2(), - ] - transf = A.Compose( - loading_transformations + downsample_train_transformations + aug_transformations + final_transformations) - return transf - - -def aggregate(x, deadzone: float, pre_mult: float, policy: str, post_mult: float, clipmargin: float, params={}): - x = x.copy() - if deadzone > 0: - x = x[(x > deadzone) | (x < -deadzone)] - if len(x) == 0: - x = np.asarray([0, ]) - if policy == 'mean': - x = np.mean(x) - x = scipy.special.expit(x * pre_mult) - x = (x - 0.5) * post_mult + 0.5 - elif policy == 'sigmean': - x = scipy.special.expit(x * pre_mult).mean() - x = (x - 0.5) * post_mult + 0.5 - elif policy == 'meanp': - pow_coeff = params.pop('p', 3) - x = np.mean(np.sign(x) * (np.abs(x) ** pow_coeff)) - x = np.sign(x) * (np.abs(x) ** (1 / pow_coeff)) - x = scipy.special.expit(x * pre_mult) - x = (x - 0.5) * post_mult + 0.5 - elif policy == 'median': - x = scipy.special.expit(np.median(x) * pre_mult) - x = (x - 0.5) * post_mult + 0.5 - elif policy == 'sigmedian': - x = np.median(scipy.special.expit(x * pre_mult)) - x = (x - 0.5) * post_mult + 0.5 - elif policy == 'maxabs': - x = np.min(x) if abs(np.min(x)) > abs(np.max(x)) else np.max(x) - x = scipy.special.expit(x * pre_mult) - x = (x - 0.5) * post_mult + 0.5 - elif policy == 'avgvoting': - x = np.mean(np.sign(x)) - x = (x * post_mult + 1) / 2 - elif policy == 'voting': - x = np.sign(np.mean(x * pre_mult)) - x = (x - 0.5) * post_mult + 0.5 - else: - raise NotImplementedError() - return np.clip(x, clipmargin, 1 - clipmargin) diff --git a/models/icpr2020dfdc/test_model.py b/models/icpr2020dfdc/test_model.py deleted file mode 100644 index ea34aaab2890f27ed3961db2bc3d21646a16ebdc..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/test_model.py +++ /dev/null @@ -1,270 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import argparse -import gc -from collections import OrderedDict -from pathlib import Path - -import albumentations as A -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import torch -import torch.nn as nn -from torch.utils.data import DataLoader -from tqdm import tqdm - -from architectures import fornet -from architectures.fornet import FeatureExtractor -from isplutils import utils, split -from isplutils.data import FrameFaceDatasetTest - - -def main(): - # Args - parser = argparse.ArgumentParser() - - parser.add_argument('--testsets', type=str, help='Testing datasets', nargs='+', choices=split.available_datasets, - required=True) - parser.add_argument('--testsplits', type=str, help='Test split', nargs='+', default=['val', 'test'], - choices=['train', 'val', 'test']) - parser.add_argument('--dfdc_faces_df_path', type=str, action='store', - help='Path to the Pandas Dataframe obtained from extract_faces.py on the DFDC dataset. ' - 'Required for training/validating on the DFDC dataset.') - parser.add_argument('--dfdc_faces_dir', type=str, action='store', - help='Path to the directory containing the faces extracted from the DFDC dataset. ' - 'Required for training/validating on the DFDC dataset.') - parser.add_argument('--ffpp_faces_df_path', type=str, action='store', - help='Path to the Pandas Dataframe obtained from extract_faces.py on the FF++ dataset. ' - 'Required for training/validating on the FF++ dataset.') - parser.add_argument('--ffpp_faces_dir', type=str, action='store', - help='Path to the directory containing the faces extracted from the FF++ dataset. ' - 'Required for training/validating on the FF++ dataset.') - - # Specify trained model path - parser.add_argument('--model_path', type=Path, help='Full path of the trained model', required=True) - - # Common params - parser.add_argument('--batch', type=int, help='Batch size to fit in GPU memory', default=128) - - parser.add_argument('--workers', type=int, help='Num workers for data loaders', default=6) - parser.add_argument('--device', type=int, help='GPU id', default=0) - - parser.add_argument('--debug', action='store_true', help='Debug flag', ) - parser.add_argument('--num_video', type=int, help='Number of real-fake videos to test') - parser.add_argument('--results_dir', type=Path, help='Output folder', - default='results/') - - parser.add_argument('--override', action='store_true', help='Override existing results', ) - - args = parser.parse_args() - - device = torch.device('cuda:{}'.format(args.device)) if torch.cuda.is_available() else torch.device('cpu') - num_workers: int = args.workers - batch_size: int = args.batch - max_num_videos_per_label: int = args.num_video # number of real-fake videos to test - model_path: Path = args.model_path - results_dir: Path = args.results_dir - debug: bool = args.debug - override: bool = args.override - test_sets = args.testsets - test_splits = args.testsplits - dfdc_df_path = args.dfdc_faces_df_path - ffpp_df_path = args.ffpp_faces_df_path - dfdc_faces_dir = args.dfdc_faces_dir - ffpp_faces_dir = args.ffpp_faces_dir - - # get arguments from the model path - face_policy = str(model_path).split('face-')[1].split('_')[0] - patch_size = int(str(model_path).split('size-')[1].split('_')[0]) - net_name = str(model_path).split('net-')[1].split('_')[0] - model_name = '_'.join(model_path.with_suffix('').parts[-2:]) - - # Load net - net_class = getattr(fornet, net_name) - - # load model - print('Loading model...') - state_tmp = torch.load(model_path, map_location='cpu') - if 'net' not in state_tmp.keys(): - state = OrderedDict({'net': OrderedDict()}) - [state['net'].update({'model.{}'.format(k): v}) for k, v in state_tmp.items()] - else: - state = state_tmp - net: FeatureExtractor = net_class().eval().to(device) - - incomp_keys = net.load_state_dict(state['net'], strict=True) - print(incomp_keys) - print('Model loaded!') - - # val loss per-frame - criterion = nn.BCEWithLogitsLoss(reduction='none') - - # Define data transformers - test_transformer = utils.get_transformer(face_policy, patch_size, net.get_normalizer(), train=False) - - # datasets and dataloaders (from train_binclass.py) - print('Loading data...') - # Check if paths for DFDC and FF++ extracted faces and DataFrames are provided - for dataset in test_sets: - if dataset.split('-')[0] == 'dfdc' and (dfdc_df_path is None or dfdc_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for DFDC faces for testing!') - elif dataset.split('-')[0] == 'ff' and (ffpp_df_path is None or ffpp_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for FF++ faces for testing!') - splits = split.make_splits(dfdc_df=dfdc_df_path, ffpp_df=ffpp_df_path, dfdc_dir=dfdc_faces_dir, - ffpp_dir=ffpp_faces_dir, dbs={'train': test_sets, 'val': test_sets, 'test': test_sets}) - train_dfs = [splits['train'][db][0] for db in splits['train']] - train_roots = [splits['train'][db][1] for db in splits['train']] - val_roots = [splits['val'][db][1] for db in splits['val']] - val_dfs = [splits['val'][db][0] for db in splits['val']] - test_dfs = [splits['test'][db][0] for db in splits['test']] - test_roots = [splits['test'][db][1] for db in splits['test']] - - # Output paths - out_folder = results_dir.joinpath(model_name) - out_folder.mkdir(mode=0o775, parents=True, exist_ok=True) - - # Samples selection - if max_num_videos_per_label and max_num_videos_per_label > 0: - dfs_out_train = [select_videos(df, max_num_videos_per_label) for df in train_dfs] - dfs_out_val = [select_videos(df, max_num_videos_per_label) for df in val_dfs] - dfs_out_test = [select_videos(df, max_num_videos_per_label) for df in test_dfs] - else: - dfs_out_train = train_dfs - dfs_out_val = val_dfs - dfs_out_test = test_dfs - - # Extractions list - extr_list = [] - # Append train and validation set first - if 'train' in test_splits: - for idx, dataset in enumerate(test_sets): - extr_list.append( - (dfs_out_train[idx], out_folder.joinpath(dataset + '_train.pkl'), train_roots[idx], dataset + ' TRAIN') - ) - if 'val' in test_splits: - for idx, dataset in enumerate(test_sets): - extr_list.append( - (dfs_out_val[idx], out_folder.joinpath(dataset + '_val.pkl'), val_roots[idx], dataset + ' VAL') - ) - if 'test' in test_splits: - for idx, dataset in enumerate(test_sets): - extr_list.append( - (dfs_out_test[idx], out_folder.joinpath(dataset + '_test.pkl'), test_roots[idx], dataset + ' TEST') - ) - - for df, df_path, df_root, tag in extr_list: - if override or not df_path.exists(): - print('\n##### PREDICT VIDEOS FROM {} #####'.format(tag)) - print('Real frames: {}'.format(sum(df['label'] == False))) - print('Fake frames: {}'.format(sum(df['label'] == True))) - print('Real videos: {}'.format(df[df['label'] == False]['video'].nunique())) - print('Fake videos: {}'.format(df[df['label'] == True]['video'].nunique())) - dataset_out = process_dataset(root=df_root, df=df, net=net, criterion=criterion, - patch_size=patch_size, - face_policy=face_policy, transformer=test_transformer, - batch_size=batch_size, - num_workers=num_workers, device=device, ) - df['score'] = dataset_out['score'].astype(np.float32) - df['loss'] = dataset_out['loss'].astype(np.float32) - print('Saving results to: {}'.format(df_path)) - df.to_pickle(str(df_path)) - - if debug: - plt.figure() - plt.title(tag) - plt.hist(df[df.label == True].score, bins=100, alpha=0.6, label='FAKE frames') - plt.hist(df[df.label == False].score, bins=100, alpha=0.6, label='REAL frames') - plt.legend() - - del (dataset_out) - del (df) - gc.collect() - - if debug: - plt.show() - - print('Completed!') - - -def process_dataset(df: pd.DataFrame, - root: str, - net: FeatureExtractor, - criterion, - patch_size: int, - face_policy: str, - transformer: A.BasicTransform, - batch_size: int, - num_workers: int, - device: torch.device, - ) -> dict: - if isinstance(device, (int, str)): - device = torch.device(device) - - dataset = FrameFaceDatasetTest( - root=root, - df=df, - size=patch_size, - scale=face_policy, - transformer=transformer, - ) - - # Preallocate - score = np.zeros(len(df)) - loss = np.zeros(len(df)) - - loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False, drop_last=False) - with torch.no_grad(): - idx0 = 0 - for batch_data in tqdm(loader): - batch_images = batch_data[0].to(device) - batch_labels = batch_data[1].to(device) - batch_samples = len(batch_images) - batch_out = net(batch_images) - batch_loss = criterion(batch_out, batch_labels) - score[idx0:idx0 + batch_samples] = batch_out.cpu().numpy()[:, 0] - loss[idx0:idx0 + batch_samples] = batch_loss.cpu().numpy()[:, 0] - idx0 += batch_samples - - out_dict = {'score': score, 'loss': loss} - return out_dict - - -def select_videos(df: pd.DataFrame, max_videos_per_label: int) -> pd.DataFrame: - """ - Select up to a maximum number of videos - :param df: DataFrame of frames. Required columns: 'video','label' - :param max_videos_per_label: maximum number of real and fake videos - :return: DataFrame of selected frames - """ - # Save random state - st0 = np.random.get_state() - # Set seed for this selection only - np.random.seed(42) - - df_fake = df[df.label == True] - fake_videos = df_fake['video'].unique() - selected_fake_videos = np.random.choice(fake_videos, min(max_videos_per_label, len(fake_videos)), replace=False) - df_selected_fake_frames = df_fake[df_fake['video'].isin(selected_fake_videos)] - - df_real = df[df.label == False] - real_videos = df_real['video'].unique() - selected_real_videos = np.random.choice(real_videos, min(max_videos_per_label, len(real_videos)), replace=False) - df_selected_real_frames = df_real[df_real['video'].isin(selected_real_videos)] - # Restore random state - np.random.set_state(st0) - - return pd.concat((df_selected_fake_frames, df_selected_real_frames), axis=0, verify_integrity=True).copy() - - -if __name__ == '__main__': - main() diff --git a/models/icpr2020dfdc/train_binclass.py b/models/icpr2020dfdc/train_binclass.py deleted file mode 100644 index 44e360fc5ff94423a78a993be055ac264e9eacdc..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/train_binclass.py +++ /dev/null @@ -1,460 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import argparse -import os -import shutil -import warnings - -import albumentations as A -import numpy as np -import pandas as pd -import torch -import torch.multiprocessing -from torchvision.transforms import ToPILImage, ToTensor - -from isplutils import utils, split - -torch.multiprocessing.set_sharing_strategy('file_system') -import torch.nn as nn -from albumentations.pytorch import ToTensorV2 -from sklearn.metrics import roc_auc_score -from tensorboardX import SummaryWriter -from torch import optim -from torch.utils.data import DataLoader -from tqdm import tqdm -from PIL import ImageChops, Image - -from architectures import fornet -from isplutils.data import FrameFaceIterableDataset, load_face - - -def main(): - # Args - parser = argparse.ArgumentParser() - parser.add_argument('--net', type=str, help='Net model class', required=True) - parser.add_argument('--traindb', type=str, help='Training datasets', nargs='+', choices=split.available_datasets, - required=True) - parser.add_argument('--valdb', type=str, help='Validation datasets', nargs='+', choices=split.available_datasets, - required=True) - parser.add_argument('--dfdc_faces_df_path', type=str, action='store', - help='Path to the Pandas Dataframe obtained from extract_faces.py on the DFDC dataset. ' - 'Required for training/validating on the DFDC dataset.') - parser.add_argument('--dfdc_faces_dir', type=str, action='store', - help='Path to the directory containing the faces extracted from the DFDC dataset. ' - 'Required for training/validating on the DFDC dataset.') - parser.add_argument('--ffpp_faces_df_path', type=str, action='store', - help='Path to the Pandas Dataframe obtained from extract_faces.py on the FF++ dataset. ' - 'Required for training/validating on the FF++ dataset.') - parser.add_argument('--ffpp_faces_dir', type=str, action='store', - help='Path to the directory containing the faces extracted from the FF++ dataset. ' - 'Required for training/validating on the FF++ dataset.') - parser.add_argument('--face', type=str, help='Face crop or scale', required=True, - choices=['scale', 'tight']) - parser.add_argument('--size', type=int, help='Train patch size', required=True) - - parser.add_argument('--batch', type=int, help='Batch size to fit in GPU memory', default=32) - parser.add_argument('--lr', type=float, default=1e-5, help='Learning rate') - parser.add_argument('--valint', type=int, help='Validation interval (iterations)', default=500) - parser.add_argument('--patience', type=int, help='Patience before dropping the LR [validation intervals]', - default=10) - parser.add_argument('--maxiter', type=int, help='Maximum number of iterations', default=20000) - parser.add_argument('--init', type=str, help='Weight initialization file') - parser.add_argument('--scratch', action='store_true', help='Train from scratch') - - parser.add_argument('--trainsamples', type=int, help='Limit the number of train samples per epoch', default=-1) - parser.add_argument('--valsamples', type=int, help='Limit the number of validation samples per epoch', - default=6000) - - parser.add_argument('--logint', type=int, help='Training log interval (iterations)', default=100) - parser.add_argument('--workers', type=int, help='Num workers for data loaders', default=6) - parser.add_argument('--device', type=int, help='GPU device id', default=0) - parser.add_argument('--seed', type=int, help='Random seed', default=0) - - parser.add_argument('--debug', action='store_true', help='Activate debug') - parser.add_argument('--suffix', type=str, help='Suffix to default tag') - - parser.add_argument('--attention', action='store_true', - help='Enable Tensorboard log of attention masks') - parser.add_argument('--log_dir', type=str, help='Directory for saving the training logs', - default='runs/binclass/') - parser.add_argument('--models_dir', type=str, help='Directory for saving the models weights', - default='weights/binclass/') - - args = parser.parse_args() - - # Parse arguments - net_class = getattr(fornet, args.net) - train_datasets = args.traindb - val_datasets = args.valdb - dfdc_df_path = args.dfdc_faces_df_path - ffpp_df_path = args.ffpp_faces_df_path - dfdc_faces_dir = args.dfdc_faces_dir - ffpp_faces_dir = args.ffpp_faces_dir - face_policy = args.face - face_size = args.size - - batch_size = args.batch - initial_lr = args.lr - validation_interval = args.valint - patience = args.patience - max_num_iterations = args.maxiter - initial_model = args.init - train_from_scratch = args.scratch - - max_train_samples = args.trainsamples - max_val_samples = args.valsamples - - log_interval = args.logint - num_workers = args.workers - device = torch.device('cuda:{:d}'.format(args.device)) if torch.cuda.is_available() else torch.device('cpu') - seed = args.seed - - debug = args.debug - suffix = args.suffix - - enable_attention = args.attention - - weights_folder = args.models_dir - logs_folder = args.log_dir - - # Random initialization - np.random.seed(seed) - torch.random.manual_seed(seed) - - # Load net - net: nn.Module = net_class().to(device) - - # Loss and optimizers - criterion = nn.BCEWithLogitsLoss() - - min_lr = initial_lr * 1e-5 - optimizer = optim.Adam(net.get_trainable_parameters(), lr=initial_lr) - lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau( - optimizer=optimizer, - mode='min', - factor=0.1, - patience=patience, - cooldown=2 * patience, - min_lr=min_lr, - ) - - tag = utils.make_train_tag(net_class=net_class, - traindb=train_datasets, - face_policy=face_policy, - patch_size=face_size, - seed=seed, - suffix=suffix, - debug=debug, - ) - - # Model checkpoint paths - bestval_path = os.path.join(weights_folder, tag, 'bestval.pth') - last_path = os.path.join(weights_folder, tag, 'last.pth') - periodic_path = os.path.join(weights_folder, tag, 'it{:06d}.pth') - - os.makedirs(os.path.join(weights_folder, tag), exist_ok=True) - - # Load model - val_loss = min_val_loss = 10 - epoch = iteration = 0 - net_state = None - opt_state = None - if initial_model is not None: - # If given load initial model - print('Loading model form: {}'.format(initial_model)) - state = torch.load(initial_model, map_location='cpu') - net_state = state['net'] - elif not train_from_scratch and os.path.exists(last_path): - print('Loading model form: {}'.format(last_path)) - state = torch.load(last_path, map_location='cpu') - net_state = state['net'] - opt_state = state['opt'] - iteration = state['iteration'] + 1 - epoch = state['epoch'] - if not train_from_scratch and os.path.exists(bestval_path): - state = torch.load(bestval_path, map_location='cpu') - min_val_loss = state['val_loss'] - if net_state is not None: - incomp_keys = net.load_state_dict(net_state, strict=False) - print(incomp_keys) - if opt_state is not None: - for param_group in opt_state['param_groups']: - param_group['lr'] = initial_lr - optimizer.load_state_dict(opt_state) - - # Initialize Tensorboard - logdir = os.path.join(logs_folder, tag) - if iteration == 0: - # If training from scratch or initialization remove history if exists - shutil.rmtree(logdir, ignore_errors=True) - - # TensorboardX instance - tb = SummaryWriter(logdir=logdir) - if iteration == 0: - dummy = torch.randn((1, 3, face_size, face_size), device=device) - dummy = dummy.to(device) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - tb.add_graph(net, [dummy, ], verbose=False) - - transformer = utils.get_transformer(face_policy=face_policy, patch_size=face_size, - net_normalizer=net.get_normalizer(), train=True) - - # Datasets and data loaders - print('Loading data') - # Check if paths for DFDC and FF++ extracted faces and DataFrames are provided - for dataset in train_datasets: - if dataset.split('-')[0] == 'dfdc' and (dfdc_df_path is None or dfdc_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for DFDC faces for training!') - elif dataset.split('-')[0] == 'ff' and (ffpp_df_path is None or ffpp_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for FF++ faces for training!') - for dataset in val_datasets: - if dataset.split('-')[0] == 'dfdc' and (dfdc_df_path is None or dfdc_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for DFDC faces for validation!') - elif dataset.split('-')[0] == 'ff' and (ffpp_df_path is None or ffpp_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for FF++ faces for validation!') - # Load splits with the make_splits function - splits = split.make_splits(dfdc_df=dfdc_df_path, ffpp_df=ffpp_df_path, dfdc_dir=dfdc_faces_dir, ffpp_dir=ffpp_faces_dir, - dbs={'train': train_datasets, 'val': val_datasets}) - train_dfs = [splits['train'][db][0] for db in splits['train']] - train_roots = [splits['train'][db][1] for db in splits['train']] - val_roots = [splits['val'][db][1] for db in splits['val']] - val_dfs = [splits['val'][db][0] for db in splits['val']] - - train_dataset = FrameFaceIterableDataset(roots=train_roots, - dfs=train_dfs, - scale=face_policy, - num_samples=max_train_samples, - transformer=transformer, - size=face_size, - ) - - val_dataset = FrameFaceIterableDataset(roots=val_roots, - dfs=val_dfs, - scale=face_policy, - num_samples=max_val_samples, - transformer=transformer, - size=face_size, - ) - - train_loader = DataLoader(train_dataset, num_workers=num_workers, batch_size=batch_size, ) - - val_loader = DataLoader(val_dataset, num_workers=num_workers, batch_size=batch_size, ) - - print('Training samples: {}'.format(len(train_dataset))) - print('Validation samples: {}'.format(len(val_dataset))) - - if len(train_dataset) == 0: - print('No training samples. Halt.') - return - - if len(val_dataset) == 0: - print('No validation samples. Halt.') - return - - stop = False - while not stop: - - # Training - optimizer.zero_grad() - - train_loss = train_num = 0 - train_pred_list = [] - train_labels_list = [] - for train_batch in tqdm(train_loader, desc='Epoch {:03d}'.format(epoch), leave=False, - total=len(train_loader) // train_loader.batch_size): - net.train() - batch_data, batch_labels = train_batch - - train_batch_num = len(batch_labels) - train_num += train_batch_num - train_labels_list.append(batch_labels.numpy().flatten()) - - train_batch_loss, train_batch_pred = batch_forward(net, device, criterion, batch_data, batch_labels) - train_pred_list.append(train_batch_pred.flatten()) - - if torch.isnan(train_batch_loss): - raise ValueError('NaN loss') - - train_loss += train_batch_loss.item() * train_batch_num - - # Optimization - train_batch_loss.backward() - optimizer.step() - optimizer.zero_grad() - - # Logging - if iteration > 0 and (iteration % log_interval == 0): - train_loss /= train_num - tb.add_scalar('train/loss', train_loss, iteration) - tb.add_scalar('lr', optimizer.param_groups[0]['lr'], iteration) - tb.add_scalar('epoch', epoch, iteration) - - # Checkpoint - save_model(net, optimizer, train_loss, val_loss, iteration, batch_size, epoch, last_path) - train_loss = train_num = 0 - - # Validation - if iteration > 0 and (iteration % validation_interval == 0): - - # Model checkpoint - save_model(net, optimizer, train_loss, val_loss, iteration, batch_size, epoch, - periodic_path.format(iteration)) - - # Train cumulative stats - train_labels = np.concatenate(train_labels_list) - train_pred = np.concatenate(train_pred_list) - train_labels_list = [] - train_pred_list = [] - - train_roc_auc = roc_auc_score(train_labels, train_pred) - tb.add_scalar('train/roc_auc', train_roc_auc, iteration) - tb.add_pr_curve('train/pr', train_labels, train_pred, iteration) - - # Validation - val_loss = validation_routine(net, device, val_loader, criterion, tb, iteration, 'val') - tb.flush() - - # LR Scheduler - lr_scheduler.step(val_loss) - - # Model checkpoint - if val_loss < min_val_loss: - min_val_loss = val_loss - save_model(net, optimizer, train_loss, val_loss, iteration, batch_size, epoch, bestval_path) - - # Attention - if enable_attention and hasattr(net, 'get_attention'): - net.eval() - # For each dataframe show the attention for a real,fake couple of frames - for df, root, sample_idx, tag in [ - (train_dfs[0], train_roots[0], train_dfs[0][train_dfs[0]['label'] == False].index[0], - 'train/att/real'), - (train_dfs[0], train_roots[0], train_dfs[0][train_dfs[0]['label'] == True].index[0], - 'train/att/fake'), - ]: - record = df.loc[sample_idx] - tb_attention(tb, tag, iteration, net, device, face_size, face_policy, - transformer, root, record) - - if optimizer.param_groups[0]['lr'] == min_lr: - print('Reached minimum learning rate. Stopping.') - stop = True - break - - iteration += 1 - - if iteration > max_num_iterations: - print('Maximum number of iterations reached') - stop = True - break - - # End of iteration - - epoch += 1 - - # Needed to flush out last events - tb.close() - - print('Completed') - - -def tb_attention(tb: SummaryWriter, - tag: str, - iteration: int, - net: nn.Module, - device: torch.device, - patch_size_load: int, - face_crop_scale: str, - val_transformer: A.BasicTransform, - root: str, - record: pd.Series, - ): - # Crop face - sample_t = load_face(record=record, root=root, size=patch_size_load, scale=face_crop_scale, - transformer=val_transformer) - sample_t_clean = load_face(record=record, root=root, size=patch_size_load, scale=face_crop_scale, - transformer=ToTensorV2()) - if torch.cuda.is_available(): - sample_t = sample_t.cuda(device) - # Transform - # Feed to net - with torch.no_grad(): - att: torch.Tensor = net.get_attention(sample_t.unsqueeze(0))[0].cpu() - att_img: Image.Image = ToPILImage()(att) - sample_img = ToPILImage()(sample_t_clean) - att_img = att_img.resize(sample_img.size, resample=Image.NEAREST).convert('RGB') - sample_att_img = ImageChops.multiply(sample_img, att_img) - sample_att = ToTensor()(sample_att_img) - tb.add_image(tag=tag, img_tensor=sample_att, global_step=iteration) - - -def batch_forward(net: nn.Module, device: torch.device, criterion, data: torch.Tensor, labels: torch.Tensor) -> ( - torch.Tensor, float, int): - data = data.to(device) - labels = labels.to(device) - out = net(data) - pred = torch.sigmoid(out).detach().cpu().numpy() - loss = criterion(out, labels) - return loss, pred - - -def validation_routine(net, device, val_loader, criterion, tb, iteration, tag: str, loader_len_norm: int = None): - net.eval() - loader_len_norm = loader_len_norm if loader_len_norm is not None else val_loader.batch_size - val_num = 0 - val_loss = 0. - pred_list = list() - labels_list = list() - for val_data in tqdm(val_loader, desc='Validation', leave=False, total=len(val_loader) // loader_len_norm): - batch_data, batch_labels = val_data - - val_batch_num = len(batch_labels) - labels_list.append(batch_labels.flatten()) - with torch.no_grad(): - val_batch_loss, val_batch_pred = batch_forward(net, device, criterion, batch_data, - batch_labels) - pred_list.append(val_batch_pred.flatten()) - val_num += val_batch_num - val_loss += val_batch_loss.item() * val_batch_num - - # Logging - val_loss /= val_num - tb.add_scalar('{}/loss'.format(tag), val_loss, iteration) - - if isinstance(criterion, nn.BCEWithLogitsLoss): - val_labels = np.concatenate(labels_list) - val_pred = np.concatenate(pred_list) - val_roc_auc = roc_auc_score(val_labels, val_pred) - tb.add_scalar('{}/roc_auc'.format(tag), val_roc_auc, iteration) - tb.add_pr_curve('{}/pr'.format(tag), val_labels, val_pred, iteration) - - return val_loss - - -def save_model(net: nn.Module, optimizer: optim.Optimizer, - train_loss: float, val_loss: float, - iteration: int, batch_size: int, epoch: int, - path: str): - path = str(path) - state = dict(net=net.state_dict(), - opt=optimizer.state_dict(), - train_loss=train_loss, - val_loss=val_loss, - iteration=iteration, - batch_size=batch_size, - epoch=epoch) - torch.save(state, path) - - -if __name__ == '__main__': - main() diff --git a/models/icpr2020dfdc/train_triplet.py b/models/icpr2020dfdc/train_triplet.py deleted file mode 100644 index 558b989f85396cb29332e54836d1c2dc4287d61f..0000000000000000000000000000000000000000 --- a/models/icpr2020dfdc/train_triplet.py +++ /dev/null @@ -1,459 +0,0 @@ -""" -Video Face Manipulation Detection Through Ensemble of CNNs - -Image and Sound Processing Lab - Politecnico di Milano - -Nicolò Bonettini -Edoardo Daniele Cannas -Sara Mandelli -Luca Bondi -Paolo Bestagini -""" -import argparse -import os -import shutil -import warnings - -import numpy as np -import torch -import torch.multiprocessing - -torch.multiprocessing.set_sharing_strategy('file_system') -import torch.nn as nn -import torch.optim as optim -from tensorboardX import SummaryWriter -from torch.utils.data import DataLoader -from tqdm import tqdm - -from architectures import tripletnet -from train_binclass import save_model, tb_attention -from isplutils.data import FrameFaceIterableDataset -from isplutils.data_siamese import FrameFaceTripletIterableDataset -from isplutils import split, utils - - -def main(): - # Args - parser = argparse.ArgumentParser() - parser.add_argument('--net', type=str, help='Net model class', required=True) - parser.add_argument('--traindb', type=str, help='Training datasets', nargs='+', choices=split.available_datasets, - required=True) - parser.add_argument('--valdb', type=str, help='Validation datasets', nargs='+', choices=split.available_datasets, - required=True) - parser.add_argument('--dfdc_faces_df_path', type=str, action='store', - help='Path to the Pandas Dataframe obtained from extract_faces.py on the DFDC dataset. ' - 'Required for training/validating on the DFDC dataset.') - parser.add_argument('--dfdc_faces_dir', type=str, action='store', - help='Path to the directory containing the faces extracted from the DFDC dataset. ' - 'Required for training/validating on the DFDC dataset.') - parser.add_argument('--ffpp_faces_df_path', type=str, action='store', - help='Path to the Pandas Dataframe obtained from extract_faces.py on the FF++ dataset. ' - 'Required for training/validating on the FF++ dataset.') - parser.add_argument('--ffpp_faces_dir', type=str, action='store', - help='Path to the directory containing the faces extracted from the FF++ dataset. ' - 'Required for training/validating on the FF++ dataset.') - parser.add_argument('--face', type=str, help='Face crop or scale', required=True, - choices=['scale', 'tight']) - parser.add_argument('--size', type=int, help='Train patch size', required=True) - - parser.add_argument('--batch', type=int, help='Batch size to fit in GPU memory', default=12) - parser.add_argument('--lr', type=float, default=1e-5, help='Learning rate') - parser.add_argument('--valint', type=int, help='Validation interval (iterations)', default=500) - parser.add_argument('--patience', type=int, help='Patience before dropping the LR [validation intervals]', - default=10) - parser.add_argument('--maxiter', type=int, help='Maximum number of iterations', default=20000) - parser.add_argument('--init', type=str, help='Weight initialization file') - parser.add_argument('--scratch', action='store_true', help='Train from scratch') - - parser.add_argument('--traintriplets', type=int, help='Limit the number of train triplets per epoch', default=-1) - parser.add_argument('--valtriplets', type=int, help='Limit the number of validation triplets per epoch', - default=2000) - - parser.add_argument('--logint', type=int, help='Training log interval (iterations)', default=100) - parser.add_argument('--workers', type=int, help='Num workers for data loaders', default=6) - parser.add_argument('--device', type=int, help='GPU device id', default=0) - parser.add_argument('--seed', type=int, help='Random seed', default=0) - - parser.add_argument('--debug', action='store_true', help='Activate debug') - parser.add_argument('--suffix', type=str, help='Suffix to default tag') - - parser.add_argument('--attention', action='store_true', - help='Enable Tensorboard log of attention masks') - parser.add_argument('--embedding', action='store_true', help='Activate embedding visualization in TensorBoard') - parser.add_argument('--embeddingint', type=int, help='Embedding visualization interval in TensorBoard', - default=5000) - - parser.add_argument('--log_dir', type=str, help='Directory for saving the training logs', - default='runs/triplet/') - parser.add_argument('--models_dir', type=str, help='Directory for saving the models weights', - default='weights/triplet/') - - args = parser.parse_args() - - # Parse arguments - net_class = getattr(tripletnet, args.net) - train_datasets = args.traindb - val_datasets = args.valdb - dfdc_df_path = args.dfdc_faces_df_path - ffpp_df_path = args.ffpp_faces_df_path - dfdc_faces_dir = args.dfdc_faces_dir - ffpp_faces_dir = args.ffpp_faces_dir - face_policy = args.face - face_size = args.size - - batch_size = args.batch - initial_lr = args.lr - validation_interval = args.valint - patience = args.patience - max_num_iterations = args.maxiter - initial_model = args.init - train_from_scratch = args.scratch - - max_train_triplets = args.traintriplets - max_val_triplets = args.valtriplets - - log_interval = args.logint - num_workers = args.workers - device = torch.device('cuda:{:d}'.format(args.device)) if torch.cuda.is_available() else torch.device('cpu') - seed = args.seed - - debug = args.debug - suffix = args.suffix - - enable_attention = args.attention - enable_embedding = args.embedding - embedding_interval = args.embeddingint - - weights_folder = args.models_dir - logs_folder = args.log_dir - - # Random initialization - np.random.seed(seed) - torch.random.manual_seed(seed) - - # Load net - net: nn.Module = net_class().to(device) - - # Loss and optimizers - criterion = nn.TripletMarginLoss() - - min_lr = initial_lr * 1e-5 - optimizer = optim.Adam(net.get_trainable_parameters(), lr=initial_lr) - lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau( - optimizer=optimizer, - mode='min', - factor=0.1, - patience=patience, - cooldown=2 * patience, - min_lr=min_lr, - ) - - tag = utils.make_train_tag(net_class=net_class, - traindb=train_datasets, - face_policy=face_policy, - patch_size=face_size, - seed=seed, - suffix=suffix, - debug=debug, - ) - - # Model checkpoint paths - bestval_path = os.path.join(weights_folder, tag, 'bestval.pth') - last_path = os.path.join(weights_folder, tag, 'last.pth') - periodic_path = os.path.join(weights_folder, tag, 'it{:06d}.pth') - - os.makedirs(os.path.join(weights_folder, tag), exist_ok=True) - - # Load model - val_loss = min_val_loss = 20 - epoch = iteration = 0 - net_state = None - opt_state = None - if initial_model is not None: - # If given load initial model - print('Loading model form: {}'.format(initial_model)) - state = torch.load(initial_model, map_location='cpu') - net_state = state['net'] - elif not train_from_scratch and os.path.exists(last_path): - print('Loading model form: {}'.format(last_path)) - state = torch.load(last_path, map_location='cpu') - net_state = state['net'] - opt_state = state['opt'] - iteration = state['iteration'] + 1 - epoch = state['epoch'] - if not train_from_scratch and os.path.exists(bestval_path): - state = torch.load(bestval_path, map_location='cpu') - min_val_loss = state['val_loss'] - if net_state is not None: - adapt_binclass_model(net_state) - incomp_keys = net.load_state_dict(net_state, strict=False) - print(incomp_keys) - if opt_state is not None: - for param_group in opt_state['param_groups']: - param_group['lr'] = initial_lr - optimizer.load_state_dict(opt_state) - - # Initialize Tensorboard - logdir = os.path.join(logs_folder, tag) - if iteration == 0: - # If training from scratch or initialization remove history if exists - shutil.rmtree(logdir, ignore_errors=True) - - # TensorboardX instance - tb = SummaryWriter(logdir=logdir) - if iteration == 0: - dummy = torch.randn((1, 3, face_size, face_size), device=device) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - tb.add_graph(net, [dummy, dummy, dummy], verbose=False) - - transformer = utils.get_transformer(face_policy=face_policy, patch_size=face_size, - net_normalizer=net.get_normalizer(), train=True) - - # Datasets and data loaders - print('Loading data') - # Check if paths for DFDC and FF++ extracted faces and DataFrames are provided - for dataset in train_datasets: - if dataset.split('-')[0] == 'dfdc' and (dfdc_df_path is None or dfdc_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for DFDC faces for training!') - elif dataset.split('-')[0] == 'ff' and (ffpp_df_path is None or ffpp_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for FF++ faces for training!') - for dataset in val_datasets: - if dataset.split('-')[0] == 'dfdc' and (dfdc_df_path is None or dfdc_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for DFDC faces for validation!') - elif dataset.split('-')[0] == 'ff' and (ffpp_df_path is None or ffpp_faces_dir is None): - raise RuntimeError('Specify DataFrame and directory for FF++ faces for validation!') - splits = split.make_splits(dfdc_df=dfdc_df_path, ffpp_df=ffpp_df_path, dfdc_dir=dfdc_faces_dir, - ffpp_dir=ffpp_faces_dir, dbs={'train': train_datasets, 'val': val_datasets}) - train_dfs = [splits['train'][db][0] for db in splits['train']] - train_roots = [splits['train'][db][1] for db in splits['train']] - val_roots = [splits['val'][db][1] for db in splits['val']] - val_dfs = [splits['val'][db][0] for db in splits['val']] - - train_dataset = FrameFaceTripletIterableDataset(roots=train_roots, - dfs=train_dfs, - scale=face_policy, - num_triplets=max_train_triplets, - transformer=transformer, - size=face_size, - ) - - val_dataset = FrameFaceTripletIterableDataset(roots=val_roots, - dfs=val_dfs, - scale=face_policy, - num_triplets=max_val_triplets, - transformer=transformer, - size=face_size, - ) - - train_loader = DataLoader(train_dataset, num_workers=num_workers, batch_size=batch_size, ) - - val_loader = DataLoader(val_dataset, num_workers=num_workers, batch_size=batch_size, ) - - print('Training triplets: {}'.format(len(train_dataset))) - print('Validation triplets: {}'.format(len(val_dataset))) - - if len(train_dataset) == 0: - print('No training triplets. Halt.') - return - - if len(val_dataset) == 0: - print('No validation triplets. Halt.') - return - - # Embedding visualization - if enable_embedding: - train_dataset_embedding = FrameFaceIterableDataset(roots=train_roots, - dfs=train_dfs, - scale=face_policy, - num_samples=64, - transformer=transformer, - size=face_size, - ) - train_loader_embedding = DataLoader(train_dataset_embedding, num_workers=num_workers, batch_size=batch_size, ) - val_dataset_embedding = FrameFaceIterableDataset(roots=val_roots, - dfs=val_dfs, - scale=face_policy, - num_samples=64, - transformer=transformer, - size=face_size, - ) - val_loader_embedding = DataLoader(val_dataset_embedding, num_workers=num_workers, batch_size=batch_size, ) - - else: - train_loader_embedding = None - val_loader_embedding = None - - stop = False - while not stop: - - # Training - optimizer.zero_grad() - - train_loss = train_num = 0 - for train_batch in tqdm(train_loader, desc='Epoch {:03d}'.format(epoch), leave=False, - total=len(train_loader) // train_loader.batch_size): - net.train() - train_batch_num = len(train_batch[0]) - train_num += train_batch_num - - train_batch_loss = batch_forward(net, device, criterion, train_batch) - - if torch.isnan(train_batch_loss): - raise ValueError('NaN loss') - - train_loss += train_batch_loss.item() * train_batch_num - - # Optimization - train_batch_loss.backward() - optimizer.step() - optimizer.zero_grad() - - # Logging - if iteration > 0 and (iteration % log_interval == 0): - train_loss /= train_num - tb.add_scalar('train/loss', train_loss, iteration) - tb.add_scalar('lr', optimizer.param_groups[0]['lr'], iteration) - tb.add_scalar('epoch', epoch, iteration) - - # Checkpoint - save_model(net, optimizer, train_loss, val_loss, iteration, batch_size, epoch, last_path) - train_loss = train_num = 0 - - # Validation - if iteration > 0 and (iteration % validation_interval == 0): - - # Validation - val_loss = validation_routine(net, device, val_loader, criterion, tb, iteration, tag='val') - tb.flush() - - # LR Scheduler - lr_scheduler.step(val_loss) - - # Model checkpoint - save_model(net, optimizer, train_loss, val_loss, iteration, batch_size, epoch, - periodic_path.format(iteration)) - if val_loss < min_val_loss: - min_val_loss = val_loss - shutil.copy(periodic_path.format(iteration), bestval_path) - - # Attention - if enable_attention and hasattr(net, 'feat_ext') and hasattr(net.feat_ext, 'get_attention'): - net.eval() - # For each dataframe show the attention for a real,fake couple of frames - - for df, root, sample_idx, tag in [ - (train_dfs[0], train_roots[0], train_dfs[0][train_dfs[0]['label'] == False].index[0], - 'train/att/real'), - (train_dfs[0], train_roots[0], train_dfs[0][train_dfs[0]['label'] == True].index[0], - 'train/att/fake'), - ]: - record = df.loc[sample_idx] - tb_attention(tb, tag, iteration, net.feat_ext, device, face_size, face_policy, - transformer, root, record) - - if optimizer.param_groups[0]['lr'] <= min_lr: - print('Reached minimum learning rate. Stopping.') - stop = True - break - - # Embedding visualization - if enable_embedding: - if iteration > 0 and (iteration % embedding_interval == 0): - embedding_routine(net=net, - device=device, - loader=train_loader_embedding, - iteration=iteration, - tb=tb, - tag=tag + '/train') - embedding_routine(net=net, - device=device, - loader=val_loader_embedding, - iteration=iteration, - tb=tb, - tag=tag + '/val') - - iteration += 1 - - if iteration > max_num_iterations: - print('Maximum number of iterations reached') - stop = True - break - - # End of iteration - - epoch += 1 - - # Needed to flush out last events - tb.close() - - print('Completed') - - -def adapt_binclass_model(net_state): - # Check that the model contains at least one key starting with feat_ext, otherwise adapt - found = False - for key in net_state: - if key.startswith('feat_ext.'): - found = True - break - if not found: - # Adapt all keys - print('Adapting keys') - keys = [k for k in net_state] - for key in keys: - net_state['feat_ext.{}'.format(key)] = net_state[key] - del net_state[key] - - -def batch_forward(net: nn.Module, device, criterion, data: tuple) -> torch.Tensor: - if torch.cuda.is_available(): - data = [i.cuda(device) for i in data] - out = net(*data) - loss = criterion(*out) - return loss - - -def validation_routine(net, device, val_loader, criterion, tb, iteration, tag): - net.eval() - - val_num = 0 - val_loss = 0. - for val_data in tqdm(val_loader, desc='Validation', leave=False, total=len(val_loader) // val_loader.batch_size): - val_batch_num = len(val_data[0]) - with torch.no_grad(): - val_batch_loss = batch_forward(net, device, criterion, val_data, ) - val_num += val_batch_num - val_loss += val_batch_loss.item() * val_batch_num - - # Logging - val_loss /= val_num - tb.add_scalar('{}/loss'.format(tag), val_loss, iteration) - - return val_loss - - -def embedding_routine(net: nn.Module, device: torch.device, loader: DataLoader, tb: SummaryWriter, iteration: int, - tag: str): - net.eval() - - labels = [] - embeddings = [] - for batch_data in loader: - batch_faces, batch_labels = batch_data - if torch.cuda.is_available(): - batch_faces = batch_faces.to(device) - with torch.no_grad(): - batch_emb = net.features(batch_faces) - labels.append(batch_labels.numpy().flatten()) - embeddings.append(torch.flatten(batch_emb.cpu(), start_dim=1).numpy()) - - labels = list(np.concatenate(labels)) - embeddings = np.concatenate(embeddings) - - # Logging - tb.add_embedding(mat=embeddings, metadata=labels, tag=tag, global_step=iteration) - - -if __name__ == '__main__': - main() diff --git a/models/model_loader.py b/models/model_loader.py index 03909810a9c354e89a1f3d5d3371e58a159705cb..d2b899b4e01f48bfdd919ed07f15e01e969d8543 100644 --- a/models/model_loader.py +++ b/models/model_loader.py @@ -1,5 +1,7 @@ from __future__ import annotations +import os +from pathlib import Path from threading import Lock from typing import Optional, Tuple @@ -21,13 +23,19 @@ class ModelLoader: cls._instance = super().__new__(cls) cls._instance._image_model = None cls._instance._image_processor = None + cls._instance._general_image_model = None + cls._instance._general_image_processor = None + cls._instance._general_image_unavailable = False cls._instance._text_pipeline = None cls._instance._multilang_text_pipeline = None cls._instance._ocr_reader = None cls._instance._face_detector = None + cls._instance._face_detector_unavailable = False cls._instance._spacy_nlp = None cls._instance._sentence_transformer = None cls._instance._efficientnet_detector = None + cls._instance._ffpp_model = None + cls._instance._ffpp_processor = None return cls._instance @classmethod @@ -48,6 +56,27 @@ class ModelLoader: logger.info("Image model loaded") return self._image_model, self._image_processor + # ---------- General AI image detector (no-face scenes / objects / art) ---------- + def load_general_image_model(self) -> Optional[Tuple[object, object]]: + if self._general_image_unavailable: + return None + if self._general_image_model is None: + try: + logger.info(f"Loading general AI image model: {settings.GENERAL_IMAGE_MODEL_ID}") + from transformers import AutoImageProcessor, AutoModelForImageClassification + + self._general_image_processor = AutoImageProcessor.from_pretrained(settings.GENERAL_IMAGE_MODEL_ID) + model = AutoModelForImageClassification.from_pretrained(settings.GENERAL_IMAGE_MODEL_ID) + model.to(settings.DEVICE) + model.eval() + self._general_image_model = model + logger.info("General AI image model loaded") + except Exception as e: # noqa: BLE001 + self._general_image_unavailable = True + logger.warning(f"General AI image model load failed: {e}") + return None + return self._general_image_model, self._general_image_processor + # ---------- Text (BERT fake-news classifier — English) ---------- def load_text_model(self): if self._text_pipeline is None: @@ -135,15 +164,25 @@ class ModelLoader: # ---------- Face detector (MediaPipe) ---------- def load_face_detector(self): + if self._face_detector_unavailable: + return None if self._face_detector is None: logger.info("Loading MediaPipe FaceMesh") - import mediapipe as mp # type: ignore + try: + import mediapipe as mp # type: ignore - self._face_detector = mp.solutions.face_mesh.FaceMesh( - static_image_mode=True, - max_num_faces=5, - min_detection_confidence=0.5, - ) + if not hasattr(mp, "solutions"): + raise ImportError("installed mediapipe package has no solutions API") + + self._face_detector = mp.solutions.face_mesh.FaceMesh( + static_image_mode=True, + max_num_faces=5, + min_detection_confidence=0.5, + ) + except Exception as exc: # noqa: BLE001 + self._face_detector_unavailable = True + logger.warning(f"MediaPipe FaceMesh unavailable: {exc}") + return None logger.info("MediaPipe FaceMesh loaded") return self._face_detector @@ -164,11 +203,104 @@ class ModelLoader: return None return self._efficientnet_detector + # ---------- FFPP-fine-tuned ViT (Phase 11.3) ---------- + def _download_ffpp_checkpoint(self) -> Optional[Path]: + repo_id = settings.FFPP_MODEL_REPO_ID.strip() + if not repo_id: + return None + + try: + from huggingface_hub import snapshot_download + except Exception as e: + logger.warning(f"huggingface_hub unavailable for FFPP checkpoint download: {e}") + return None + + try: + revision = settings.FFPP_MODEL_REVISION.strip() or "main" + token = os.getenv("HF_TOKEN") or None + logger.info(f"Downloading FFPP ViT checkpoint from Hub: {repo_id}@{revision}") + snapshot_dir = snapshot_download( + repo_id=repo_id, + repo_type="model", + revision=revision, + allow_patterns=["config.json", "model.safetensors"], + token=token, + ) + checkpoint_dir = Path(snapshot_dir) + if not (checkpoint_dir / "config.json").exists(): + logger.warning( + f"Downloaded FFPP checkpoint from {repo_id} but config.json is missing" + ) + return None + return checkpoint_dir + except Exception as e: + logger.warning(f"FFPP checkpoint download failed from {repo_id}: {e}") + return None + + def load_ffpp_model(self) -> Optional[Tuple[object, object]]: + """Lazy-load the FaceForensics++ fine-tuned ViT from a local checkpoint. + + The checkpoint directory was exported from Colab with only + `model.safetensors` + `config.json` (no preprocessor_config.json), so the + image processor is loaded from the base google/vit-base-patch16-224-in21k + — this matches the processor used during training. + + Returns None if disabled or the checkpoint is missing. + """ + if not settings.FFPP_ENABLED: + return None + if self._ffpp_model is not None: + return self._ffpp_model, self._ffpp_processor + + configured_path = Path(settings.FFPP_MODEL_PATH) + repo_root = Path(__file__).resolve().parent.parent.parent + candidates = [configured_path] if configured_path.is_absolute() else [ + (repo_root / configured_path).resolve(), + (Path.cwd() / configured_path).resolve(), + (repo_root / "trained_models").resolve(), + ] + ckpt_path = next((p for p in candidates if (p / "config.json").exists()), candidates[0]) + + if not (ckpt_path / "config.json").exists(): + downloaded = self._download_ffpp_checkpoint() + if downloaded is not None: + ckpt_path = downloaded + else: + tried = ", ".join(str(p) for p in candidates) + logger.warning(f"FFPP ViT checkpoint not found. Tried: {tried} — skipping") + return None + + try: + from transformers import AutoImageProcessor, AutoModelForImageClassification + + logger.info(f"Loading FFPP ViT model from {ckpt_path}") + processor = AutoImageProcessor.from_pretrained(settings.FFPP_BASE_PROCESSOR_ID) + model = AutoModelForImageClassification.from_pretrained(str(ckpt_path)) + model.to(settings.DEVICE) + model.eval() + self._ffpp_model = model + self._ffpp_processor = processor + logger.info("FFPP ViT model loaded") + return self._ffpp_model, self._ffpp_processor + except Exception as e: + logger.warning(f"FFPP ViT load failed (continuing without it): {e}") + return None + # ---------- Preload ---------- def preload_phase1(self) -> None: """Preload only what Phase 1 needs (image model).""" self.load_image_model() + def is_ready(self) -> bool: + """Phase 19.5 — readiness signal for /health/ready. + + When PRELOAD_MODELS is enabled, readiness = image model loaded. + Otherwise the loader constructs successfully → ready (lazy-load on demand). + """ + if settings.PRELOAD_MODELS: + return self._image_model is not None + return True + def get_model_loader() -> ModelLoader: return ModelLoader.get_instance() diff --git a/news_lookup.py b/news_lookup.py deleted file mode 100644 index 8831afb27b3e5d852cf6c2838c8bd96ceca8420d..0000000000000000000000000000000000000000 --- a/news_lookup.py +++ /dev/null @@ -1,242 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import List, Optional, Tuple -from urllib.parse import urlparse - -import httpx -from loguru import logger - -from config import settings -from schemas.common import ContradictingEvidence, TrustedSource, TruthOverride - -# Trusted news domains — higher relevance boost -TRUSTED_DOMAINS = { - "reuters.com": 1.0, "apnews.com": 1.0, "bbc.com": 1.0, "bbc.co.uk": 1.0, - "theguardian.com": 0.95, "nytimes.com": 0.95, "washingtonpost.com": 0.95, - "cnn.com": 0.9, "npr.org": 0.95, "aljazeera.com": 0.9, - "thehindu.com": 0.9, "indianexpress.com": 0.9, "ndtv.com": 0.85, - "hindustantimes.com": 0.85, "pti.news": 0.95, -} - -# Fact-check / contradiction sources -FACTCHECK_DOMAINS = { - "factcheck.org", "snopes.com", "politifact.com", "fullfact.org", - "reuters.com/fact-check", "apnews.com/hub/ap-fact-check", - "factly.in", "altnews.in", "boomlive.in", "vishvasnews.com", -} - -# Domains eligible for truth-override (weight >= 0.9 per BUILD_PLAN spec) -_HIGH_TRUST_DOMAINS = {d for d, w in TRUSTED_DOMAINS.items() if w >= 0.9} - -# Thresholds per BUILD_PLAN §13.2 -_OVERRIDE_SIMILARITY_THRESHOLD = 0.6 -_OVERRIDE_FAKE_PROB_CAP = 0.15 -_OVERRIDE_FAKE_PROB_MULTIPLIER = 0.3 - - -@dataclass -class NewsLookupResult: - trusted_sources: List[TrustedSource] - contradicting_evidence: List[ContradictingEvidence] - total_articles: int - truth_override: Optional[TruthOverride] = None - - -def _domain_of(url: str) -> str: - try: - return urlparse(url).netloc.lower().replace("www.", "") - except Exception: - return "" - - -def _is_factcheck(url: str, title: str) -> bool: - dom = _domain_of(url) - if any(fc in dom for fc in FACTCHECK_DOMAINS): - return True - tl = (title or "").lower() - return any(kw in tl for kw in ("fact check", "fact-check", "debunked", "false claim", "misleading", "hoax")) - - -def _relevance(url: str) -> float: - dom = _domain_of(url) - for td, score in TRUSTED_DOMAINS.items(): - if td in dom: - return score - return 0.5 - - -def _is_high_trust(url: str) -> bool: - dom = _domain_of(url) - return any(ht in dom for ht in _HIGH_TRUST_DOMAINS) - - -def _compute_truth_override( - input_text: str, - trusted_sources: List[TrustedSource], - current_fake_prob: float, -) -> Optional[TruthOverride]: - """Check if any high-trust source corroborates the input text at >= 0.6 cosine similarity. - - Per BUILD_PLAN §13.2: - - Compute cosine similarity between input_text and each trusted-source headline+description - - If ≥ 1 high-trust source (weight ≥ 0.9) has similarity ≥ 0.6 → apply fake_prob *= 0.3, cap at 0.15 - """ - if not input_text or not trusted_sources: - return None - - # Filter to high-trust sources only - high_trust = [s for s in trusted_sources if _is_high_trust(s.url)] - if not high_trust: - return None - - # Lazy-load sentence-transformer - from models.model_loader import get_model_loader - st_model = get_model_loader().load_sentence_transformer() - if st_model is None: - return None - - try: - import numpy as np - - # Encode input text and all high-trust headlines - source_texts = [ - f"{s.title}" for s in high_trust - ] - all_texts = [input_text[:512]] + source_texts - - embeddings = st_model.encode(all_texts, convert_to_numpy=True, normalize_embeddings=True) - query_vec = embeddings[0] # (D,) - source_vecs = embeddings[1:] # (N, D) - - # Cosine similarity — already normalized, so dot product = cosine similarity - similarities = np.dot(source_vecs, query_vec) - - best_idx = int(np.argmax(similarities)) - best_sim = float(similarities[best_idx]) - best_source = high_trust[best_idx] - - logger.info( - f"Truth-override: best similarity={best_sim:.3f} " - f"source={best_source.source_name} url={best_source.url}" - ) - - if best_sim >= _OVERRIDE_SIMILARITY_THRESHOLD: - new_fake_prob = min( - current_fake_prob * _OVERRIDE_FAKE_PROB_MULTIPLIER, - _OVERRIDE_FAKE_PROB_CAP, - ) - logger.info( - f"Truth-override APPLIED: fake_prob {current_fake_prob:.3f} → {new_fake_prob:.3f}" - ) - return TruthOverride( - applied=True, - source_url=best_source.url, - source_name=best_source.source_name, - similarity=round(best_sim, 4), - fake_prob_before=round(current_fake_prob, 4), - fake_prob_after=round(new_fake_prob, 4), - ) - - return TruthOverride( - applied=False, - source_url=best_source.url, - source_name=best_source.source_name, - similarity=round(best_sim, 4), - fake_prob_before=round(current_fake_prob, 4), - fake_prob_after=round(current_fake_prob, 4), - ) - - except Exception as e: - logger.warning(f"Truth-override computation failed: {e}") - return None - - -async def _fetch(q: str, country: Optional[str]) -> list[dict]: - target_country = country or "in" - params = {"apikey": settings.NEWS_API_KEY, "q": q, "language": "en", "size": 10, "country": "in"} - - try: - async with httpx.AsyncClient(timeout=8.0) as c: - r = await c.get(settings.NEWS_API_BASE_URL, params=params) - r.raise_for_status() - return (r.json() or {}).get("results") or [] - except Exception as e: - logger.warning(f"News lookup failed: {e}") - return [] - - -async def search_news( - keywords: List[str], - limit: int = 6, - country: Optional[str] = None, -) -> List[TrustedSource]: - """Back-compat simple form — returns trusted sources only.""" - result = await search_news_full(keywords, limit=limit, country=country) - return result.trusted_sources - - -async def search_news_full( - keywords: List[str], - limit: int = 6, - country: Optional[str] = None, - original_text: Optional[str] = None, - current_fake_prob: float = 0.5, -) -> NewsLookupResult: - """Full news lookup with truth-override support. - - Args: - keywords: NER-extracted or frequency-extracted keywords to search. - limit: Max sources to return. - country: Country code for newsdata.io. - original_text: Input text to compare against headlines for truth-override. - current_fake_prob: Current fake probability — may be adjusted by truth-override. - """ - if not settings.NEWS_API_KEY or not keywords: - return NewsLookupResult([], [], 0) - - q = " ".join(keywords[:4]) - articles = await _fetch(q, country) - - seen: set[str] = set() - trusted: List[TrustedSource] = [] - contradictions: List[ContradictingEvidence] = [] - - for art in articles: - url = art.get("link") or "" - if not url or url in seen: - continue - seen.add(url) - - title = art.get("title") or "" - dom = _domain_of(url) - src_name = art.get("source_id") or dom or "news" - - if _is_factcheck(url, title): - contradictions.append(ContradictingEvidence( - source_name=src_name, title=title, url=url, type="fact_check", - )) - continue - - trusted.append(TrustedSource( - source_name=src_name, - title=title, - url=url, - published_at=art.get("pubDate"), - relevance_score=_relevance(url), - )) - - trusted.sort(key=lambda s: -s.relevance_score) - trusted = trusted[:limit] - - # ── Phase 13.2: Truth-override ── - truth_override = None - if original_text and trusted: - truth_override = _compute_truth_override(original_text, trusted, current_fake_prob) - - return NewsLookupResult( - trusted_sources=trusted, - contradicting_evidence=contradictions[:limit], - total_articles=len(articles), - truth_override=truth_override, - ) diff --git a/report.html b/report.html deleted file mode 100644 index 17189b7194f885ccb8c690ae7accfe93578780a9..0000000000000000000000000000000000000000 --- a/report.html +++ /dev/null @@ -1,367 +0,0 @@ -<!DOCTYPE html> -<html> -<head> - <meta charset="utf-8" /> - <title>DeepShield Analysis Report — {{ analysis_id }} - - - - - {# ── Header ── #} - - - - - -
▮ DeepShield - Analysis Report  ·  ID: {{ analysis_id }}
- Media: {{ media_type | upper }}  ·  Generated: {{ generated_at }} -
- - {# ── Verdict ── #} -

Verdict

- - - - - {% if donut_b64 %} - - {% endif %} - -
-
{{ verdict.authenticity_score }}
-
/ 100
- 		-
{{ verdict.label }}
-
Severity: {{ verdict.severity }}
-
Model: {{ verdict.model_label }}  ({{ '%.1f' | format(verdict.model_confidence * 100) }}% confidence)
- 		- score donut -
- - {# ── LLM Explanation ── #} - {% if llm_summary and llm_summary.paragraph %} -

AI Explanation

-
-

{{ llm_summary.paragraph }}

- {% if llm_summary.bullets %} -
    - {% for b in llm_summary.bullets %}
  • {{ b }}
    • {% endfor %} -
- {% endif %} - {% if llm_summary.model_used %} -
via {{ llm_summary.model_used }}
- {% endif %} -
- {% endif %} - - {# ══════════ IMAGE ══════════ #} - {% if media_type == 'image' %} - - {# EXIF #} - {% if explainability.exif %} -

EXIF Metadata

- - - {% if explainability.exif.make %} - - {% endif %} - {% if explainability.exif.model %} - - {% endif %} - {% if explainability.exif.datetime_original %} - - {% endif %} - {% if explainability.exif.software %} - - - {% endif %} - {% if explainability.exif.lens_model %} - - {% endif %} - {% if explainability.exif.gps_info %} - - {% endif %} - - - - -
FieldValueTrust Signal
Camera Make{{ explainability.exif.make }}+real
Camera Model{{ explainability.exif.model }}
Date Taken{{ explainability.exif.datetime_original }}+real
Software{{ explainability.exif.software }}{% if 'photoshop' in explainability.exif.software | lower %}+fake{% endif %}
Lens Model{{ explainability.exif.lens_model }}
GPS{{ explainability.exif.gps_info }}
Trust adjustment - {% if explainability.exif.trust_adjustment > 0 %} - +{{ explainability.exif.trust_adjustment }} (fake signal) - {% elif explainability.exif.trust_adjustment < 0 %} - {{ explainability.exif.trust_adjustment }} (real signal) - {% else %} - neutral - {% endif %} -
- {% endif %} - - {# Artifact indicators #} - {% if explainability.artifact_indicators %} -

Artifact Indicators

- - - {% for ind in explainability.artifact_indicators %} - - - - - - - {% endfor %} -
TypeSeverityConfidenceDescription
{{ ind.type }}{{ ind.severity }}{{ '%.0f' | format(ind.confidence * 100) }}%{{ ind.description }}
- {% else %} -

Artifact Indicators

-
No artifacts detected.
- {% endif %} - - {# VLM Detailed Breakdown #} - {% if explainability.vlm_breakdown %} -

Detailed Breakdown

- {% if explainability.vlm_breakdown.model_used %} -
Scored by {{ explainability.vlm_breakdown.model_used }}
- {% endif %} - - - {% set bd = explainability.vlm_breakdown %} - {% for comp_key, comp_label in [ - ('facial_symmetry', 'Facial Symmetry'), - ('skin_texture', 'Skin Texture'), - ('lighting_consistency', 'Lighting Consistency'), - ('background_coherence', 'Background Coherence'), - ('anatomy_hands_eyes', 'Anatomy / Hands & Eyes'), - ('context_objects', 'Context & Objects') - ] %} - {% set comp = bd[comp_key] %} - {% set sc2 = comp.score if comp else 75 %} - {% set bar_cls = 'vlm-real' if sc2 >= 70 else ('vlm-warn' if sc2 >= 40 else 'vlm-fake') %} - - - - - - - {% endfor %} -
ComponentScoreBarNotes
{{ comp_label }}{{ sc2 }}/100 - - - - {{ comp.notes if comp else '' }}
- {% endif %} - - {% endif %}{# end image #} - - {# ══════════ VIDEO ══════════ #} - {% if media_type == 'video' %} -

Frame-Level Analysis

- - - - - - - - -
MetricValue
Frames sampled{{ explainability.num_frames_sampled }}
Frames with face{{ explainability.num_face_frames }}
Suspicious frames{{ explainability.num_suspicious_frames }}
Mean suspicious prob{{ '%.1f' | format(explainability.mean_suspicious_prob * 100) }}%
Max suspicious prob{{ '%.1f' | format(explainability.max_suspicious_prob * 100) }}%
Insufficient faces{{ explainability.insufficient_faces }}
- {% endif %} - - {# ══════════ TEXT ══════════ #} - {% if media_type == 'text' %} - - {# Language + truth-override #} - {% if explainability.detected_language and explainability.detected_language != 'en' %} -

Language

-
Detected: {{ explainability.detected_language | upper }} — analysed via multilingual model
- {% endif %} - {% if explainability.truth_override and explainability.truth_override.applied %} -
- Truth-override applied. - Corroborated by {{ explainability.truth_override.source_name }} - ({{ '%.0f' | format(explainability.truth_override.similarity * 100) }}% similarity). - Fake probability reduced from {{ '%.1f' | format(explainability.truth_override.fake_prob_before * 100) }}% - to {{ '%.1f' | format(explainability.truth_override.fake_prob_after * 100) }}%. -
- {% endif %} - -

Text Classification

- - - - - - - - - -
MetricValue
Fake probability{{ '%.1f' | format(explainability.fake_probability * 100) }}%
Top label{{ explainability.top_label }}
Sensationalism score{{ explainability.sensationalism.score }}/100 ({{ explainability.sensationalism.level }})
Exclamations{{ explainability.sensationalism.exclamation_count }}
ALL CAPS words{{ explainability.sensationalism.caps_word_count }}
Clickbait matches{{ explainability.sensationalism.clickbait_matches }}
Emotional words{{ explainability.sensationalism.emotional_word_count }}
- - {% if explainability.manipulation_indicators %} -

Manipulation Indicators ({{ explainability.manipulation_indicators | length }})

- - - {% for m in explainability.manipulation_indicators %} - - - - - - {% endfor %} -
PatternSeverityMatched text
{{ m.pattern_type }}{{ m.severity }}{{ m.matched_text }}
- {% endif %} - - {% if explainability.keywords %} -

Extracted Keywords

-
{% for kw in explainability.keywords %}{{ kw }}{% endfor %}
- {% endif %} - - {% endif %}{# end text #} - - {# ══════════ SCREENSHOT ══════════ #} - {% if media_type == 'screenshot' %} - - {% if explainability.detected_language and explainability.detected_language != 'en' %} -
Detected language: {{ explainability.detected_language | upper }}
- {% endif %} - {% if explainability.truth_override and explainability.truth_override.applied %} -
- Truth-override applied. {{ explainability.truth_override.source_name }} - ({{ '%.0f' | format(explainability.truth_override.similarity * 100) }}% similarity) -
- {% endif %} - -

Extracted Text

-
{{ explainability.ocr_boxes | length }} OCR regions detected
- - -
{{ explainability.extracted_text }}
- -

Analysis Summary

- - - - - - -
MetricValue
Fake probability{{ '%.1f' | format(explainability.fake_probability * 100) }}%
Sensationalism{{ explainability.sensationalism.score }}/100 ({{ explainability.sensationalism.level }})
Suspicious phrases{{ explainability.suspicious_phrases | length }}
Layout anomalies{{ explainability.layout_anomalies | length }}
- - {% if explainability.suspicious_phrases %} -

Suspicious Phrases

- - - {% for p in explainability.suspicious_phrases %} - - - - - - {% endfor %} -
TextPatternSeverity
{{ p.text }}{{ p.pattern_type }}{{ p.severity }}
- {% endif %} - - {% endif %}{# end screenshot #} - - {# ══════════ SOURCES (all types) ══════════ #} - {% if trusted_sources %} -

Trusted Source Cross-Reference ({{ trusted_sources | length }})

- - - {% for s in trusted_sources %} - - - - - - {% endfor %} -
SourceTitleRelevance
{{ s.source_name }}{{ s.title }}{{ '%.0f' | format(s.relevance_score * 100) }}%
- {% endif %} - - {% if contradicting_evidence %} -

Contradicting Evidence ({{ contradicting_evidence | length }})

- - - {% for c in contradicting_evidence %} - - {% endfor %} -
SourceTitleType
{{ c.source_name }}{{ c.title }}{{ c.type }}
- {% endif %} - - {# ══════════ PROCESSING ══════════ #} -

Processing Summary

-
Model: {{ processing_summary.model_used }}  ·  Duration: {{ processing_summary.total_duration_ms }} ms
-
{{ processing_summary.stages_completed | join(' → ') }}
- - {# ══════════ FOOTER ══════════ #} -
- DeepShield Responsible-AI Notice. {{ responsible_ai_notice }}
- Generated {{ generated_at }}. Report expires in 1 hour. - AI-assisted analysis — cross-check with trusted sources before sharing. -
- - - diff --git a/report_service.py b/report_service.py deleted file mode 100644 index 154503b6179adbcd19ba924682e3f3649d8b0cc6..0000000000000000000000000000000000000000 --- a/report_service.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -import base64 -import json -import os -import time -import uuid -from datetime import datetime, timedelta, timezone -from io import BytesIO -from pathlib import Path -from typing import Any, Optional - -from jinja2 import Environment, FileSystemLoader, select_autoescape -from loguru import logger -from xhtml2pdf import pisa # type: ignore - -from config import settings -from db.models import AnalysisRecord, Report - -TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "templates" - -_env = Environment( - loader=FileSystemLoader(str(TEMPLATES_DIR)), - autoescape=select_autoescape(["html", "xml"]), -) - - -def _score_class(score: int) -> str: - if score >= 70: - return "real" - if score >= 40: - return "warn" - return "fake" - - -def _ensure_dir() -> Path: - p = Path(settings.REPORT_DIR) - p.mkdir(parents=True, exist_ok=True) - return p - - -def _make_donut_chart(score: int, score_cls: str) -> str: - """Render authenticity score as a donut chart PNG; return base64 or '' on failure.""" - try: - import matplotlib # type: ignore - matplotlib.use("Agg") - import matplotlib.pyplot as plt # type: ignore - - color_map = {"real": "#43A047", "warn": "#FB8C00", "fake": "#E53935"} - color = color_map.get(score_cls, "#6B7280") - - fig, ax = plt.subplots(figsize=(2.2, 2.2), dpi=96) - sizes = [score, 100 - score] - wedge_colors = [color, "#F3F4F6"] - ax.pie(sizes, colors=wedge_colors, startangle=90, - wedgeprops=dict(width=0.42, edgecolor="white", linewidth=1)) - ax.text(0, 0, str(score), ha="center", va="center", - fontsize=20, fontweight="bold", color=color) - ax.set_aspect("equal") - plt.tight_layout(pad=0.05) - - buf = BytesIO() - fig.savefig(buf, format="png", bbox_inches="tight", transparent=True) - plt.close(fig) - buf.seek(0) - return base64.b64encode(buf.read()).decode() - except Exception as e: - logger.debug(f"Donut chart skipped: {e}") - return "" - - -def _extract_llm_summary(analysis_json: dict) -> dict | None: - """Extract llm_summary from either top-level or inside explainability (images).""" - top = analysis_json.get("llm_summary") - if top: - return top - return (analysis_json.get("explainability") or {}).get("llm_summary") - - -def render_html(analysis_json: dict) -> str: - score = analysis_json.get("verdict", {}).get("authenticity_score", 50) - sc = _score_class(score) - donut_b64 = _make_donut_chart(score, sc) - llm_summary = _extract_llm_summary(analysis_json) - expl: dict[str, Any] = analysis_json.get("explainability") or {} - - tmpl = _env.get_template("report.html") - return tmpl.render( - analysis_id=analysis_json.get("analysis_id", ""), - media_type=analysis_json.get("media_type", "unknown"), - verdict=analysis_json.get("verdict", {}), - explainability=expl, - trusted_sources=analysis_json.get("trusted_sources", []), - contradicting_evidence=analysis_json.get("contradicting_evidence", []), - processing_summary=analysis_json.get("processing_summary", {}), - responsible_ai_notice=analysis_json.get( - "responsible_ai_notice", - "AI-based analysis may not be 100% accurate.", - ), - score_class=sc, - generated_at=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"), - donut_b64=donut_b64, - llm_summary=llm_summary, - ) - - -def html_to_pdf(html: str, out_path: Path) -> None: - with open(out_path, "wb") as f: - result = pisa.CreatePDF(html, dest=f) - if result.err: - raise RuntimeError(f"xhtml2pdf failed with {result.err} errors") - - -def generate_report(record: AnalysisRecord) -> Path: - out_dir = _ensure_dir() - filename = f"deepshield_{record.id}_{uuid.uuid4().hex[:8]}.pdf" - out_path = out_dir / filename - - data = json.loads(record.result_json) - html = render_html(data) - html_to_pdf(html, out_path) - logger.info(f"Report generated id={record.id} path={out_path} size={out_path.stat().st_size}B") - return out_path - - -def create_report_row(analysis_id: int, path: Path) -> Report: - return Report( - analysis_id=analysis_id, - file_path=str(path), - expires_at=datetime.utcnow() + timedelta(seconds=settings.REPORT_TTL_SECONDS), - ) - - -def cleanup_expired(now: Optional[datetime] = None) -> int: - """Delete expired PDFs from disk. Returns count deleted.""" - now = now or datetime.utcnow() - d = Path(settings.REPORT_DIR) - if not d.exists(): - return 0 - deleted = 0 - ttl = timedelta(seconds=settings.REPORT_TTL_SECONDS) - for f in d.glob("*.pdf"): - try: - mtime = datetime.utcfromtimestamp(f.stat().st_mtime) - if now - mtime > ttl: - f.unlink() - deleted += 1 - except OSError as e: - logger.warning(f"Cleanup failed for {f}: {e}") - if deleted: - logger.info(f"Cleaned up {deleted} expired reports") - return deleted diff --git a/requirements.txt b/requirements.txt index 4acf6728d4f096cc9888c7892a0bc014ee46a3c5..61538bd8186cb8c9c47cc5f603f418aaf62982d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,3 +55,9 @@ xhtml2pdf==0.2.16 # === Phase 8: Auth === email-validator==2.2.0 + +# === Phase 15: Security Hardening === +slowapi==0.1.9 + +# === Phase 17: Video Pipeline v2 === +ffmpeg-python==0.2.0 # Python wrapper for ffmpeg subprocess (audio extraction) diff --git a/router.py b/router.py deleted file mode 100644 index 478bb749be5a898755712262b76c49db8bd1257f..0000000000000000000000000000000000000000 --- a/router.py +++ /dev/null @@ -1,10 +0,0 @@ -from fastapi import APIRouter - -from api.v1 import analyze, auth, health, history, report - -api_router = APIRouter(prefix="/api/v1") -api_router.include_router(health.router) -api_router.include_router(analyze.router) -api_router.include_router(report.router) -api_router.include_router(auth.router) -api_router.include_router(history.router) diff --git a/schemas/analyze.py b/schemas/analyze.py index 252939f3c6a78c228a52137684a7511b0692426c..e4d8c59fc2d4d9d8cbd521d38e361624e5057b94 100644 --- a/schemas/analyze.py +++ b/schemas/analyze.py @@ -50,6 +50,8 @@ class TextExplainability(BaseModel): class TextAnalysisResponse(BaseModel): analysis_id: str record_id: int = 0 + cached: bool = False + thumbnail_url: str | None = None media_type: str = "text" timestamp: str verdict: Verdict @@ -99,6 +101,8 @@ class ScreenshotExplainability(BaseModel): class ScreenshotAnalysisResponse(BaseModel): analysis_id: str record_id: int = 0 + cached: bool = False + thumbnail_url: str | None = None media_type: str = "screenshot" timestamp: str verdict: Verdict @@ -117,8 +121,13 @@ class ImageExplainability(BaseModel): ela_base64: str = "" boxes_base64: str = "" heatmap_status: str = "success" # success | failed | degraded + # Persistent file URLs — available on reload (not excluded from DB storage) + heatmap_url: str | None = None + ela_url: str | None = None + boxes_url: str | None = None artifact_indicators: List[ArtifactIndicator] = [] exif: ExifSummary | None = None + no_face_analysis: dict | None = None llm_summary: LLMExplainabilitySummary | None = None vlm_breakdown: VLMBreakdown | None = None @@ -134,6 +143,16 @@ class FrameAnalysisOut(BaseModel): scored: bool = False +class AudioExplainability(BaseModel): + audio_authenticity_score: float = 100.0 + has_audio: bool = False + duration_s: float = 0.0 + silence_ratio: float = 0.0 + spectral_variance: float = 0.0 + rms_consistency: float = 0.0 + notes: str = "" + + class VideoExplainability(BaseModel): num_frames_sampled: int num_face_frames: int = 0 @@ -144,11 +163,20 @@ class VideoExplainability(BaseModel): insufficient_faces: bool = False suspicious_timestamps: List[float] = [] frames: List[FrameAnalysisOut] = [] + # Phase 17.1 — temporal consistency + temporal_score: float | None = None + optical_flow_variance: float | None = None + flicker_score: float | None = None + blink_rate_anomaly: bool | None = None + # Phase 17.2 — audio deepfake detection + audio: AudioExplainability | None = None class VideoAnalysisResponse(BaseModel): analysis_id: str record_id: int = 0 + cached: bool = False + thumbnail_url: str | None = None media_type: str = "video" timestamp: str verdict: Verdict @@ -165,6 +193,9 @@ class VideoAnalysisResponse(BaseModel): class ImageAnalysisResponse(BaseModel): analysis_id: str record_id: int = 0 + cached: bool = False + thumbnail_url: str | None = None + media_path: str | None = None media_type: str = "image" timestamp: str verdict: Verdict diff --git a/schemas/common.py b/schemas/common.py index 8baa8018a569f18e0e145dcd0d36c9748c18f251..4a86e74cb922c8882ac9dbc92417cdf5eaffdc92 100644 --- a/schemas/common.py +++ b/schemas/common.py @@ -4,6 +4,8 @@ from typing import List, Optional from pydantic import BaseModel, ConfigDict, Field +ANALYSIS_CACHE_VERSION = "2026-04-26-accuracy-v3" + class Verdict(BaseModel): model_config = ConfigDict(protected_namespaces=()) @@ -26,6 +28,7 @@ class TrustedSource(BaseModel): source_name: str title: str url: str + description: Optional[str] = None published_at: Optional[str] = None relevance_score: float = Field(ge=0.0, le=1.0) @@ -87,3 +90,5 @@ class ProcessingSummary(BaseModel): total_duration_ms: int model_used: str models_used: List[str] = [] # all models that contributed (ensemble) + analysis_version: str = ANALYSIS_CACHE_VERSION + calibrator_applied: bool = False diff --git a/scoring.py b/scoring.py deleted file mode 100644 index eec7009e2d63204fd7952bc3e0d30afb561dacc1..0000000000000000000000000000000000000000 --- a/scoring.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -from typing import Tuple - -TRUST_SCALE = [ - (0, 20, "Very Likely Fake", "critical"), - (21, 40, "Likely Fake", "danger"), - (41, 60, "Possibly Manipulated", "warning"), - (61, 80, "Likely Real", "positive"), - (81, 100, "Very Likely Real", "safe"), -] - - -def compute_authenticity_score(model_confidence: float, label: str) -> int: - """Map (confidence, label) to 0-100 authenticity score. - Real-ish labels give high score; fake-ish labels give low score. - """ - label_l = label.lower() - fake_tokens = ("fake", "deepfake", "manipulated", "ai", "generated", "synthetic") - if any(tok in label_l for tok in fake_tokens): - score = (1.0 - float(model_confidence)) * 100.0 - else: - score = float(model_confidence) * 100.0 - return int(round(max(0.0, min(100.0, score)))) - - -def get_verdict_label(score: int) -> Tuple[str, str]: - for lo, hi, label, severity in TRUST_SCALE: - if lo <= score <= hi: - return label, severity - return "Unknown", "warning" - - -def get_score_color(score: int) -> str: - """Linear interpolate Red (#E53935) → Amber (#FFA726) → Green (#43A047).""" - def lerp(a: int, b: int, t: float) -> int: - return int(round(a + (b - a) * t)) - - score = max(0, min(100, score)) - if score <= 50: - t = score / 50.0 - r, g, b = lerp(0xE5, 0xFF, t), lerp(0x39, 0xA7, t), lerp(0x35, 0x26, t) - else: - t = (score - 50) / 50.0 - r, g, b = lerp(0xFF, 0x43, t), lerp(0xA7, 0xA0, t), lerp(0x26, 0x47, t) - return f"#{r:02X}{g:02X}{b:02X}" diff --git a/screenshot_service.py b/screenshot_service.py deleted file mode 100644 index ae5aa3eed6986c0f9a940965bcdd5dfaedfa0dfb..0000000000000000000000000000000000000000 --- a/screenshot_service.py +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import List, Tuple - -import numpy as np -from loguru import logger -from PIL import Image - -from models.model_loader import get_model_loader - - -@dataclass -class OCRBox: - text: str - bbox: List[List[int]] # 4 points [[x,y],...] - confidence: float - - -@dataclass -class SuspiciousPhrase: - text: str - bbox: List[List[int]] - pattern_type: str - severity: str - description: str - - -@dataclass -class LayoutAnomaly: - type: str # misalignment / font_mismatch / uneven_spacing - severity: str - description: str - confidence: float - - -def run_ocr(pil_img: Image.Image) -> List[OCRBox]: - reader = get_model_loader().load_ocr_engine() - arr = np.array(pil_img.convert("RGB")) - results = reader.readtext(arr, detail=1, paragraph=False) - out: List[OCRBox] = [] - for bbox, text, conf in results: - out.append(OCRBox( - text=str(text), - bbox=[[int(p[0]), int(p[1])] for p in bbox], - confidence=float(conf), - )) - logger.info(f"OCR extracted {len(out)} text regions") - return out - - -def extract_full_text(boxes: List[OCRBox]) -> str: - return " ".join(b.text for b in boxes if b.text.strip()) - - -def map_phrases_to_boxes(boxes: List[OCRBox], manipulation_indicators) -> List[SuspiciousPhrase]: - """Map each manipulation indicator to the OCR box whose text contains it.""" - out: List[SuspiciousPhrase] = [] - for mi in manipulation_indicators: - needle = mi.matched_text.lower() - for b in boxes: - if needle in b.text.lower(): - out.append(SuspiciousPhrase( - text=mi.matched_text, - bbox=b.bbox, - pattern_type=mi.pattern_type, - severity=mi.severity, - description=mi.description, - )) - break - return out - - -def detect_layout_anomalies(boxes: List[OCRBox]) -> List[LayoutAnomaly]: - """Heuristic layout checks on OCR bboxes.""" - out: List[LayoutAnomaly] = [] - if len(boxes) < 3: - return out - - heights = [] - x_lefts = [] - for b in boxes: - pts = b.bbox - ys = [p[1] for p in pts] - xs = [p[0] for p in pts] - heights.append(max(ys) - min(ys)) - x_lefts.append(min(xs)) - - h_arr = np.array(heights, dtype=float) - if h_arr.mean() > 0: - cv_h = float(h_arr.std() / h_arr.mean()) - if cv_h > 0.7: - out.append(LayoutAnomaly( - type="font_mismatch", - severity="medium" if cv_h < 1.2 else "high", - description=f"High variance in text heights (cv={cv_h:.2f}) — mixed fonts/sizes possible", - confidence=min(cv_h / 1.5, 1.0), - )) - - x_arr = np.array(x_lefts, dtype=float) - if x_arr.std() > 0 and len(x_arr) > 4: - clustered = sum(1 for x in x_arr if abs(x - np.median(x_arr)) < 20) - align_ratio = clustered / len(x_arr) - if align_ratio < 0.4: - out.append(LayoutAnomaly( - type="misalignment", - severity="low", - description=f"Only {align_ratio*100:.0f}% of text blocks share left-alignment — unusual layout", - confidence=1.0 - align_ratio, - )) - - if len(boxes) >= 4: - tops = sorted([min(p[1] for p in b.bbox) for b in boxes]) - gaps = np.diff(tops) - gaps = gaps[gaps > 0] - if len(gaps) >= 3 and gaps.mean() > 0: - cv_g = float(gaps.std() / gaps.mean()) - if cv_g > 1.5: - out.append(LayoutAnomaly( - type="uneven_spacing", - severity="low", - description=f"Irregular vertical spacing between text blocks (cv={cv_g:.2f})", - confidence=min(cv_g / 2.5, 1.0), - )) - - return out diff --git a/scripts/export_onnx.py b/scripts/export_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..102d6c184b49af71bd81cb0db340eae1e32a18c2 --- /dev/null +++ b/scripts/export_onnx.py @@ -0,0 +1,111 @@ +"""P3: Export EfficientNetAutoAttB4 to ONNX for 2-3× CPU inference speedup. + +Exports the model to backend/models/efficientnet_autoattb4_dfdc.onnx. +After export, set EFFICIENTNET_ONNX_PATH in .env to enable ONNX inference. + +Requirements (install first): + pip install onnx onnxruntime + +Usage: + .venv/Scripts/python.exe scripts/export_onnx.py [--validate] + +The --validate flag runs a quick numerical comparison between PyTorch and ONNX +outputs on a random face-shaped input to verify the export is correct. +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +ONNX_OUT = Path(__file__).resolve().parent.parent / "models" / "efficientnet_autoattb4_dfdc.onnx" + + +def export(out_path: Path, opset: int = 17) -> None: + print("Loading EfficientNetAutoAttB4…") + from services.efficientnet_service import EfficientNetDetector + det = EfficientNetDetector() + net = det.net.eval().cpu() + + dummy = torch.zeros(1, 3, 224, 224) + + print(f"Exporting to ONNX (opset {opset})…") + out_path.parent.mkdir(parents=True, exist_ok=True) + torch.onnx.export( + net, + dummy, + str(out_path), + opset_version=opset, + input_names=["face"], + output_names=["logit"], + dynamic_axes={"face": {0: "batch"}, "logit": {0: "batch"}}, + do_constant_folding=True, + ) + size_mb = out_path.stat().st_size / 1024 / 1024 + print(f"Saved: {out_path} ({size_mb:.1f} MB)") + + +def validate(out_path: Path) -> None: + try: + import onnxruntime as ort + except ImportError: + print("onnxruntime not installed — skipping validation. pip install onnxruntime") + return + + print("Validating ONNX output vs PyTorch…") + from services.efficientnet_service import EfficientNetDetector + det = EfficientNetDetector() + net = det.net.eval().cpu() + + dummy = torch.randn(1, 3, 224, 224) + with torch.inference_mode(): + pt_out = net(dummy).numpy() + + sess = ort.InferenceSession(str(out_path), providers=["CPUExecutionProvider"]) + ort_out = sess.run(None, {"face": dummy.numpy()})[0] + + max_diff = float(np.abs(pt_out - ort_out).max()) + print(f" Max absolute diff PyTorch vs ONNX: {max_diff:.6f}") + if max_diff < 1e-4: + print(" [PASS] Outputs match within tolerance") + else: + print(" [WARN] Outputs differ more than 1e-4 — inspect export") + + # Benchmark. + import time + N = 20 + t0 = time.perf_counter() + for _ in range(N): + sess.run(None, {"face": dummy.numpy()}) + ort_ms = (time.perf_counter() - t0) / N * 1000 + + t0 = time.perf_counter() + with torch.inference_mode(): + for _ in range(N): + net(dummy) + pt_ms = (time.perf_counter() - t0) / N * 1000 + + print(f" PyTorch: {pt_ms:.1f} ms/img | ONNX: {ort_ms:.1f} ms/img | speedup: {pt_ms/ort_ms:.2f}×") + print(f"\nTo enable ONNX inference, add to .env:\n EFFICIENTNET_ONNX_PATH={out_path}") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Export EfficientNetAutoAttB4 to ONNX") + parser.add_argument("--out", type=Path, default=ONNX_OUT, help="Output .onnx file path") + parser.add_argument("--opset", type=int, default=17, help="ONNX opset version (default 17)") + parser.add_argument("--validate", action="store_true", help="Compare ONNX vs PyTorch outputs and benchmark") + args = parser.parse_args() + + export(args.out, opset=args.opset) + if args.validate: + validate(args.out) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fit_calibrator.py b/scripts/fit_calibrator.py new file mode 100644 index 0000000000000000000000000000000000000000..53478a3f1561670aada5f57cff2c9de1cd6bdcbf --- /dev/null +++ b/scripts/fit_calibrator.py @@ -0,0 +1,167 @@ +"""Isotonic calibration for EfficientNetAutoAttB4 — §7.6 of MERGE_PLAN. + +Fits sklearn.isotonic.IsotonicRegression on EfficientNetAutoAttB4's raw sigmoid +outputs and persists the result to backend/models/efficientnet_calibrator.pkl. + +Usage: + .venv/Scripts/python.exe scripts/fit_calibrator.py --real PATH --fake PATH [--val-split 0.2] + +Directory layout expected: + --real path/to/real/faces/ (JPEG/PNG face images, labelled 0) + --fake path/to/fake/faces/ (JPEG/PNG deepfake images, labelled 1) + +FFPP c40 example (from Phase 11.1 Colab download): + --real training/datasets/ffpp/c40/real/ + --fake training/datasets/ffpp/c40/fake/ + +The script: + 1. Runs EfficientNet inference on all images (face detection → sigmoid score). + 2. Splits into train/val (stratified, default 80/20). + 3. Fits IsotonicRegression(out_of_bounds='clip') on training split. + 4. Evaluates on val split: accuracy, real→fake FPR, fake→real FNR. + 5. Saves calibrator to backend/models/efficientnet_calibrator.pkl. + +Run time: ~5 min on a 50-200 image set on CPU. +""" +from __future__ import annotations + +import argparse +import pickle +import sys +from pathlib import Path + +import numpy as np +from loguru import logger + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp"} +CALIBRATOR_OUT = Path(__file__).resolve().parent.parent / "models" / "efficientnet_calibrator.pkl" + + +def collect_images(directory: Path) -> list[Path]: + return sorted(p for p in directory.rglob("*") if p.suffix.lower() in IMAGE_EXTS) + + +def score_images(det, paths: list[Path]) -> list[float]: + """Run EfficientNet on each image; return raw sigmoid scores (-1 sentinel for no-face).""" + from PIL import Image + scores = [] + for i, p in enumerate(paths): + try: + pil = Image.open(p).convert("RGB") + except Exception as e: + logger.warning(f"Cannot open {p}: {e}") + scores.append(-1.0) + continue + import numpy as np_inner + img_np = np_inner.array(pil) + frame_data = det.face_extractor.process_image(img=img_np) + faces = frame_data.get("faces", []) + if not faces: + scores.append(-1.0) + else: + face_t = det._face_tensor(faces[0]) + import torch + logit = det.raw_logit(face_t) + from scipy.special import expit + scores.append(float(expit(logit))) + if (i + 1) % 10 == 0: + print(f" scored {i + 1}/{len(paths)}", end="\r") + print() + return scores + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fit isotonic calibrator for EfficientNetAutoAttB4") + parser.add_argument("--real", required=True, type=Path, help="Directory of real face images (label=0)") + parser.add_argument("--fake", required=True, type=Path, help="Directory of deepfake images (label=1)") + parser.add_argument("--val-split", type=float, default=0.2, help="Fraction held out for validation (default 0.2)") + parser.add_argument("--out", type=Path, default=CALIBRATOR_OUT, help="Output pkl path") + args = parser.parse_args() + + if not args.real.is_dir(): + print(f"ERROR: --real must be a directory: {args.real}") + return 1 + if not args.fake.is_dir(): + print(f"ERROR: --fake must be a directory: {args.fake}") + return 1 + + real_paths = collect_images(args.real) + fake_paths = collect_images(args.fake) + if not real_paths: + print(f"ERROR: No images found in {args.real}") + return 1 + if not fake_paths: + print(f"ERROR: No images found in {args.fake}") + return 1 + print(f"Found {len(real_paths)} real | {len(fake_paths)} fake images") + + print("Loading EfficientNetDetector (weights cached after first run)…") + from services.efficientnet_service import EfficientNetDetector + # Load without applying existing calibrator — we are building a new one. + det = EfficientNetDetector(calibrator_path=Path("/dev/null")) + + print("Scoring real images…") + real_scores = score_images(det, real_paths) + print("Scoring fake images…") + fake_scores = score_images(det, fake_paths) + + # Build arrays, drop no-face sentinels. + r_scores = np.array([s for s in real_scores if s >= 0]) + f_scores = np.array([s for s in fake_scores if s >= 0]) + r_labels = np.zeros(len(r_scores)) + f_labels = np.ones(len(f_scores)) + + X = np.concatenate([r_scores, f_scores]) + y = np.concatenate([r_labels, f_labels]) + print(f"\nUsable samples: {len(r_scores)} real | {len(f_scores)} fake") + print(f"No-face dropped: {sum(s < 0 for s in real_scores)} real | {sum(s < 0 for s in fake_scores)} fake") + + if len(X) < 10: + print("ERROR: Too few usable samples (<10) to fit a calibrator.") + return 1 + + # Stratified train/val split. + from sklearn.model_selection import train_test_split + X_tr, X_val, y_tr, y_val = train_test_split( + X, y, test_size=args.val_split, stratify=y, random_state=42 + ) + print(f"Split: {len(X_tr)} train | {len(X_val)} val") + + print("Fitting IsotonicRegression…") + from sklearn.isotonic import IsotonicRegression + cal = IsotonicRegression(out_of_bounds="clip") + cal.fit(X_tr.reshape(-1, 1), y_tr) + + # Evaluate on val set. + y_pred_raw = (X_val >= 0.5).astype(int) + y_pred_cal = (cal.predict(X_val.reshape(-1, 1)) >= 0.5).astype(int) + + def metrics(y_true, y_pred, tag): + acc = (y_true == y_pred).mean() * 100 + real_mask = y_true == 0 + fpr = (y_pred[real_mask] == 1).mean() * 100 if real_mask.sum() > 0 else 0.0 + fake_mask = y_true == 1 + fnr = (y_pred[fake_mask] == 0).mean() * 100 if fake_mask.sum() > 0 else 0.0 + print(f" [{tag}] acc={acc:.1f}% real→fake FPR={fpr:.1f}% fake→real FNR={fnr:.1f}%") + return acc, fpr + + print("\nValidation metrics:") + acc_raw, fpr_raw = metrics(y_val, y_pred_raw, "raw ") + acc_cal, fpr_cal = metrics(y_val, y_pred_cal, "calibrated") + + # Gate G3: ≥88% accuracy, ≤8% FPR. + g3_pass = acc_cal >= 88.0 and fpr_cal <= 8.0 + print(f"\n Gate G3: {'PASS ✓' if g3_pass else 'FAIL ✗'} (need acc≥88%, FPR≤8%)") + + args.out.parent.mkdir(parents=True, exist_ok=True) + with args.out.open("wb") as f: + pickle.dump(cal, f, protocol=pickle.HIGHEST_PROTOCOL) + print(f"\nCalibrator saved → {args.out}") + print("Restart the backend server for the calibrator to take effect.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_efficientnet_load.py b/scripts/test_efficientnet_load.py new file mode 100644 index 0000000000000000000000000000000000000000..1ccddfb262f57ad7c4d348ad820ebc9433db2011 --- /dev/null +++ b/scripts/test_efficientnet_load.py @@ -0,0 +1,86 @@ +"""G1/G2 smoke test — EfficientNetAutoAttB4 + BlazeFace load and basic inference. + +Gate G1: model loads on cold start without crash. +Gate G2: BlazeFace detects ≥1 face on a synthetic face image. + +Run from backend/: + .venv/Scripts/python.exe scripts/test_efficientnet_load.py +""" +from __future__ import annotations + +import sys +import time +import urllib.request +from pathlib import Path + +import numpy as np +import psutil + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + + +def main() -> int: + proc = psutil.Process() + ram_before = proc.memory_info().rss / 1024 / 1024 + + print("=== G1: EfficientNetAutoAttB4 load ===") + t0 = time.perf_counter() + try: + from services.efficientnet_service import EfficientNetDetector + det = EfficientNetDetector() + elapsed = time.perf_counter() - t0 + print(f" [PASS] model loaded in {elapsed:.1f}s") + except Exception as e: + print(f" [FAIL] {e}") + return 1 + + ram_after = proc.memory_info().rss / 1024 / 1024 + print(f" RAM delta: +{ram_after - ram_before:.0f} MB (total: {ram_after:.0f} MB)") + + print("\n=== G2: BlazeFace face detection ===") + # Download a small real portrait image for face detection test. + url = "https://upload.wikimedia.org/wikipedia/commons/thumb/1/14/Gatto_europeo4.jpg/320px-Gatto_europeo4.jpg" + face_url = "https://thispersondoesnotexist.com/" + print(f" fetching test face from: {face_url}") + try: + from PIL import Image + import io + req = urllib.request.Request(face_url, headers={"User-Agent": "DeepShield/1.0"}) + with urllib.request.urlopen(req, timeout=15) as r: + data = r.read() + pil = Image.open(io.BytesIO(data)).convert("RGB") + img_np = np.array(pil) + frame_data = det.face_extractor.process_image(img=img_np) + faces = frame_data.get("faces", []) + if faces: + print(f" [PASS] BlazeFace detected {len(faces)} face(s)") + else: + print(" [WARN] BlazeFace detected 0 faces on test image — G2 inconclusive (network or image issue)") + except Exception as e: + print(f" [WARN] face detection test skipped: {e}") + + print("\n=== G1b: detect_image on synthetic noise (no-face path) ===") + try: + from PIL import Image as PILImage + noise = PILImage.fromarray(np.random.randint(0, 255, (224, 224, 3), dtype=np.uint8)) + result = det.detect_image(noise) + if result.get("error") == "no_face": + print(" [PASS] no-face path returns gracefully") + else: + print(f" [INFO] unexpected result on noise image: {result}") + except Exception as e: + print(f" [FAIL] detect_image raised: {e}") + return 1 + + print("\n=== Memory gate (G8 check) ===") + ram_final = proc.memory_info().rss / 1024 / 1024 + threshold_mb = 2500 + status = "PASS" if ram_final < threshold_mb else "WARN" + print(f" [{status}] RSS={ram_final:.0f} MB (threshold {threshold_mb} MB)") + + print("\nAll G1 gates passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_image_classify.py b/scripts/test_image_classify.py index d38a0b667ed32057de0a08738f461535328648f5..504a6b2125bef29977b49cafe8c5d4e313de9c72 100644 --- a/scripts/test_image_classify.py +++ b/scripts/test_image_classify.py @@ -45,7 +45,11 @@ def main() -> int: print(f" [{ind.severity.upper():6s}] {ind.type}: {ind.description} (conf {ind.confidence:.2f})") print("\nGenerating Grad-CAM heatmap\u2026") - heatmap_url = generate_heatmap_base64(pil) + heatmap_url, heatmap_source = generate_heatmap_base64(pil) + print(f" heatmap source: {heatmap_source}") + if not heatmap_url: + print(" no heatmap (no face or fallback)") + return 0 header, b64 = heatmap_url.split(",", 1) out_path = Path(__file__).resolve().parent.parent / "heatmap_smoketest.png" out_path.write_bytes(base64.b64decode(b64)) diff --git a/services/artifact_detector.py b/services/artifact_detector.py index bb05435afa5d80def094f961af0390679659358e..4eaf9470702cf9b845a24b5d4d6abfa0e6e154f2 100644 --- a/services/artifact_detector.py +++ b/services/artifact_detector.py @@ -44,18 +44,20 @@ def detect_gan_hf_artifact(pil_img: Image.Image) -> ArtifactIndicator | None: total = float(mag.sum() + 1e-9) hf = float(mag[hf_mask].sum()) - ratio = hf / total # typically 0.05–0.20 for natural photos + ratio = hf / total - # normalize to [0,1] suspiciousness - score = max(0.0, min(1.0, (ratio - 0.10) / 0.20)) + # Passport/ID portraits often have strong fine detail from hair, fabric, + # sharpening, and JPEG ringing. Treat this as a weak forensic signal + # unless it is extreme; the classifier ensemble remains authoritative. + score = max(0.0, min(1.0, (ratio - 0.24) / 0.28)) sev = _severity_from_score(score) return ArtifactIndicator( type="gan_artifact", severity=sev, description=( f"High-frequency energy ratio {ratio:.3f} — " - + ("elevated HF energy consistent with GAN/diffusion outputs" if score > 0.4 - else "natural frequency falloff") + + ("elevated fine-detail/compression energy; review with model score" if score > 0.4 + else "within expected range for a natural photo") ), confidence=float(score), ) @@ -138,11 +140,11 @@ def detect_face_based_artifacts(pil_img: Image.Image) -> List[ArtifactIndicator] """ results: List[ArtifactIndicator] = [] try: - import mediapipe as mp # type: ignore - from models.model_loader import get_model_loader detector = get_model_loader().load_face_detector() + if detector is None: + return results rgb = np.asarray(pil_img.convert("RGB")) h, w = rgb.shape[:2] mp_result = detector.process(rgb) diff --git a/services/audio_service.py b/services/audio_service.py new file mode 100644 index 0000000000000000000000000000000000000000..04e1668c44dd9ea4f9141dd261935bb8587e8396 --- /dev/null +++ b/services/audio_service.py @@ -0,0 +1,202 @@ +"""Phase 17.2 — Audio Deepfake Detection. + +Extracts the audio track from a video with ffmpeg, then applies signal-processing +heuristics (silence ratio, spectral centroid variance, RMS consistency) to produce +an audio_authenticity_score (0–100, higher = more natural/authentic). + +AI-generated speech typically exhibits: + - Near-zero silence between words (no natural breath pauses) + - Very low spectral-centroid variance (monotone formant trajectory) + - Unnaturally consistent RMS energy across voiced frames +""" +from __future__ import annotations + +import os +import subprocess +import tempfile +from dataclasses import dataclass +from typing import Optional + +import numpy as np +from loguru import logger + + +@dataclass +class AudioAnalysis: + audio_authenticity_score: float # 0–100 + has_audio: bool + duration_s: float + silence_ratio: float # fraction of 25ms frames below RMS threshold + spectral_variance: float # normalised std of spectral centroid + rms_consistency: float # 1 – normalised std of voiced-frame RMS + notes: str = "" + + +# --------------------------------------------------------------------------- +# ffmpeg extraction +# --------------------------------------------------------------------------- + +def _extract_audio_wav(video_path: str, out_path: str) -> bool: + """Extract mono 16 kHz WAV from *video_path* into *out_path* via ffmpeg.""" + try: + result = subprocess.run( + [ + "ffmpeg", "-y", "-i", video_path, + "-vn", "-acodec", "pcm_s16le", + "-ar", "16000", "-ac", "1", + out_path, + ], + capture_output=True, + timeout=60, + ) + return result.returncode == 0 and os.path.getsize(out_path) > 0 + except (FileNotFoundError, subprocess.TimeoutExpired, OSError) as exc: + logger.warning(f"ffmpeg audio extraction failed: {exc}") + return False + + +# --------------------------------------------------------------------------- +# Signal-processing analysis +# --------------------------------------------------------------------------- + +def _analyse_wav(wav_path: str) -> AudioAnalysis: + try: + from scipy.io import wavfile # scipy already in requirements + sr, data = wavfile.read(wav_path) + except Exception as exc: # noqa: BLE001 + logger.warning(f"WAV read failed: {exc}") + return AudioAnalysis( + audio_authenticity_score=50.0, has_audio=True, + duration_s=0.0, silence_ratio=0.0, + spectral_variance=0.0, rms_consistency=0.0, + notes="wav_read_failed", + ) + + # Flatten stereo → mono + if data.ndim > 1: + data = data[:, 0] + + data = data.astype(np.float32) / (np.iinfo(np.int16).max + 1) + duration_s = float(len(data) / sr) + + if duration_s < 0.1: + return AudioAnalysis( + audio_authenticity_score=50.0, has_audio=True, + duration_s=round(duration_s, 3), silence_ratio=1.0, + spectral_variance=0.0, rms_consistency=0.0, + notes="too_short", + ) + + # --- 25ms framing --- + frame_len = max(1, int(sr * 0.025)) + hop_len = max(1, frame_len // 2) + frames = [ + data[i: i + frame_len] + for i in range(0, len(data) - frame_len, hop_len) + ] + if not frames: + return AudioAnalysis( + audio_authenticity_score=50.0, has_audio=True, + duration_s=round(duration_s, 3), silence_ratio=1.0, + spectral_variance=0.0, rms_consistency=0.0, + notes="no_frames", + ) + + rms_vals = np.array([np.sqrt(np.mean(f ** 2)) for f in frames]) + + # Silence ratio + SILENCE_THRESH = 0.01 + silence_ratio = float(np.mean(rms_vals < SILENCE_THRESH)) + + # Spectral centroid variance + freqs = np.fft.rfftfreq(frame_len, d=1.0 / sr) + centroids: list[float] = [] + for frame in frames: + spec = np.abs(np.fft.rfft(frame)) + total = float(np.sum(spec)) + if total < 1e-9: + continue + centroids.append(float(np.dot(freqs, spec) / total)) + + spec_var = ( + float(np.std(centroids) / (np.mean(centroids) + 1e-6)) + if centroids else 0.0 + ) + + # RMS consistency on voiced frames + voiced = rms_vals[rms_vals >= SILENCE_THRESH] + if len(voiced) > 0: + rms_consistency = float( + 1.0 - min(1.0, np.std(voiced) / (np.mean(voiced) + 1e-6)) + ) + else: + rms_consistency = 0.5 + + # --- Heuristic scoring --- + # Silence score: natural speech has moderate pauses (0.1–0.6). + # < 0.05 → no pauses (suspicious); > 0.85 → near-silent (unclear). + if silence_ratio < 0.05: + silence_score = 55.0 + elif silence_ratio > 0.85: + silence_score = 50.0 + else: + silence_score = 100.0 + + # Spectral variance score: natural formant motion gives spec_var > 0.25. + spec_score = min(100.0, spec_var * 250.0) + + # RMS consistency: > 0.92 = unnaturally even (TTS/vocoder artifact). + rms_score = 55.0 if rms_consistency > 0.92 else 100.0 + + audio_score = float( + 0.30 * silence_score + 0.50 * spec_score + 0.20 * rms_score + ) + audio_score = max(20.0, min(100.0, audio_score)) + + logger.info( + f"Audio: dur={duration_s:.1f}s silence={silence_ratio:.2f} " + f"spec_var={spec_var:.4f} rms_cons={rms_consistency:.4f} " + f"→ audio_score={audio_score:.1f}" + ) + + return AudioAnalysis( + audio_authenticity_score=round(audio_score, 2), + has_audio=True, + duration_s=round(duration_s, 2), + silence_ratio=round(silence_ratio, 4), + spectral_variance=round(spec_var, 4), + rms_consistency=round(rms_consistency, 4), + ) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def analyze_audio(video_path: str) -> Optional[AudioAnalysis]: + """Extract and analyse the audio track from *video_path*. + + Returns an AudioAnalysis dataclass, or None if no audio track is present + or if ffmpeg is unavailable. + """ + tmp_wav: Optional[str] = None + try: + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fh: + tmp_wav = fh.name + + if not _extract_audio_wav(video_path, tmp_wav): + logger.info("No audio track found or ffmpeg unavailable — skipping audio analysis") + return None + + return _analyse_wav(tmp_wav) + + except Exception as exc: # noqa: BLE001 + logger.warning(f"Audio analysis error: {exc}") + return None + + finally: + if tmp_wav and os.path.exists(tmp_wav): + try: + os.unlink(tmp_wav) + except OSError: + pass diff --git a/services/dedup_cache.py b/services/dedup_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..81e8b9164c8a88fcc5f014c63ed05741ba318998 --- /dev/null +++ b/services/dedup_cache.py @@ -0,0 +1,65 @@ +"""Phase 19.1 — SHA-256 media dedup cache. + +Looks up a prior AnalysisRecord by content hash within CACHE_TTL_DAYS, and +returns the cached payload so repeated uploads of the same file skip the +expensive analyzer pipelines. +""" + +from __future__ import annotations + +import json +import os +from datetime import datetime, timedelta + +from loguru import logger +from sqlalchemy.orm import Session + +from db.models import AnalysisRecord +from schemas.common import ANALYSIS_CACHE_VERSION + +CACHE_TTL_DAYS = int(os.environ.get("CACHE_TTL_DAYS", "30")) + + +def lookup_cached( + db: Session, + *, + media_hash: str, + media_type: str, + user_id: int | None, +) -> AnalysisRecord | None: + """Return a cached AnalysisRecord for this hash+type if within TTL. + + We scope the cache by user when the user is signed in (their own history + should return their own cached record) and globally when anonymous. + """ + if not media_hash: + return None + cutoff = datetime.utcnow() - timedelta(days=CACHE_TTL_DAYS) + q = ( + db.query(AnalysisRecord) + .filter( + AnalysisRecord.media_hash == media_hash, + AnalysisRecord.media_type == media_type, + AnalysisRecord.created_at >= cutoff, + ) + .order_by(AnalysisRecord.created_at.desc()) + ) + if user_id is not None: + return q.filter(AnalysisRecord.user_id == user_id).first() + return q.filter(AnalysisRecord.user_id.is_(None)).first() + + +def cached_payload(record: AnalysisRecord) -> dict | None: + """Decode stored result_json and stamp the cached flag.""" + try: + payload = json.loads(record.result_json) + except Exception as e: # noqa: BLE001 + logger.warning(f"cached payload decode failed for record {record.id}: {e}") + return None + summary = payload.get("processing_summary") or {} + if summary.get("analysis_version") != ANALYSIS_CACHE_VERSION: + logger.info(f"cache stale for record {record.id}: analysis_version mismatch") + return None + payload["cached"] = True + payload["record_id"] = record.id + return payload diff --git a/services/efficientnet_service.py b/services/efficientnet_service.py index c8212bbb04f47262b51199f4c792f0c207bf210d..e36337371c81c2525445b4dabbbe59f6e9233e1a 100644 --- a/services/efficientnet_service.py +++ b/services/efficientnet_service.py @@ -6,6 +6,7 @@ import sys from pathlib import Path from typing import List, Optional +import cv2 import numpy as np import torch from loguru import logger @@ -99,6 +100,69 @@ class EfficientNetDetector: f"| calibrator={'yes' if self.calibrator_applied else 'no'}" ) + def _crop_with_margin( + self, + img_array: np.ndarray, + x0: int, + y0: int, + x1: int, + y1: int, + margin: float = 0.22, + ) -> Optional[np.ndarray]: + h, w = img_array.shape[:2] + bw = max(1, x1 - x0) + bh = max(1, y1 - y0) + pad = int(max(bw, bh) * margin) + x0 = max(0, x0 - pad) + y0 = max(0, y0 - pad) + x1 = min(w, x1 + pad) + y1 = min(h, y1 + pad) + if x1 <= x0 + 8 or y1 <= y0 + 8: + return None + return img_array[y0:y1, x0:x1] + + def _fallback_face_crop(self, img_array: np.ndarray) -> Optional[np.ndarray]: + """Fallback face crop for real-world still photos where BlazeFace misses. + + BlazeFace is tuned for the ICPR2020 pipeline. Real phone portraits can be + large, soft, or vertically framed, so use MediaPipe/Haar only to recover a + crop and still score it with the same EfficientNet model. + """ + try: + from models.model_loader import get_model_loader + + detector = get_model_loader().load_face_detector() + mp_result = detector.process(img_array) if detector is not None else None + if mp_result is not None and getattr(mp_result, "multi_face_landmarks", None): + landmarks = mp_result.multi_face_landmarks[0].landmark + h, w = img_array.shape[:2] + xs = [lm.x * w for lm in landmarks] + ys = [lm.y * h for lm in landmarks] + crop = self._crop_with_margin( + img_array, + int(min(xs)), + int(min(ys)), + int(max(xs)), + int(max(ys)), + ) + if crop is not None: + return crop + except Exception as exc: # noqa: BLE001 + logger.debug(f"MediaPipe fallback face crop failed: {exc}") + + try: + gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY) + cascade_path = cv2.data.haarcascades + "haarcascade_frontalface_default.xml" + cascade = cv2.CascadeClassifier(cascade_path) + faces = cascade.detectMultiScale(gray, scaleFactor=1.08, minNeighbors=4, minSize=(32, 32)) + if len(faces) == 0: + return None + x, y, w, h = max(faces, key=lambda box: box[2] * box[3]) + return self._crop_with_margin(img_array, int(x), int(y), int(x + w), int(y + h)) + except Exception as exc: # noqa: BLE001 + logger.debug(f"OpenCV fallback face crop failed: {exc}") + return None + def _face_tensor(self, face_np: np.ndarray) -> torch.Tensor: """Apply albumentations transform to a cropped face array and return a CHW tensor.""" result = self.transf(image=face_np) @@ -140,15 +204,20 @@ class EfficientNetDetector: frame_data = self.face_extractor.process_image(img=img_array) faces: list = frame_data.get("faces", []) + detector_used = "blazeface" if not faces: - logger.debug("EfficientNetDetector.detect_image: no face detected") - return { - "error": "no_face", - "score": None, - "result": None, - "model": f"{self.model_name}_{self.train_db}", - "calibrator_applied": False, - } + fallback_crop = self._fallback_face_crop(img_array) + if fallback_crop is None: + logger.debug("EfficientNetDetector.detect_image: no face detected") + return { + "error": "no_face", + "score": None, + "result": None, + "model": f"{self.model_name}_{self.train_db}", + "calibrator_applied": False, + } + faces = [fallback_crop] + detector_used = "mediapipe_or_haar_fallback" face_t = self._face_tensor(faces[0]) with torch.inference_mode(): @@ -162,6 +231,7 @@ class EfficientNetDetector: "model": f"{self.model_name}_{self.train_db}", "error": None, "calibrator_applied": self.calibrator_applied, + "face_detector": detector_used, } def detect_video_frames(self, frames: List[np.ndarray]) -> dict: @@ -182,6 +252,10 @@ class EfficientNetDetector: faces: list = frame_data.get("faces", []) if faces: face_tensors.append(self._face_tensor(faces[0])) + else: + fallback_crop = self._fallback_face_crop(frame_rgb) + if fallback_crop is not None: + face_tensors.append(self._face_tensor(fallback_crop)) if not face_tensors: logger.debug("EfficientNetDetector.detect_video_frames: no faces in any frame") diff --git a/services/ela_service.py b/services/ela_service.py index e937502d11a21c347c224611a155047d8c88bbfc..f978814f1e2eeb48a1e5ab3d324a5148a0966e9f 100644 --- a/services/ela_service.py +++ b/services/ela_service.py @@ -10,7 +10,6 @@ from __future__ import annotations import base64 import io -import cv2 import numpy as np from loguru import logger from PIL import Image diff --git a/services/exif_service.py b/services/exif_service.py index 61c69ef39ff455182b5b1f0ab151f0d5aac8bbff..205fd5d4c20f50eba8b3a5c3f350f8d4a20abc8b 100644 --- a/services/exif_service.py +++ b/services/exif_service.py @@ -41,6 +41,8 @@ def _decode_gps(gps_info: dict) -> Optional[str]: lat = _to_decimal(gps_info.get(2, (0, 0, 0)), gps_info.get(1, "N")) lon = _to_decimal(gps_info.get(4, (0, 0, 0)), gps_info.get(3, "E")) + if abs(lat) < 1e-9 and abs(lon) < 1e-9: + return None return f"{lat:.6f}, {lon:.6f}" except Exception: return None @@ -105,22 +107,29 @@ def extract_exif(pil_img: Image.Image, raw_bytes: bytes) -> ExifSummary: has_camera_meta = summary.make and summary.model and summary.datetime_original if has_camera_meta: - adjustment -= 15 + adjustment -= 8 reasons.append("valid camera metadata (Make/Model/DateTime)") if summary.gps_info: - adjustment -= 5 + adjustment -= 2 reasons.append("GPS coordinates present") + # Lens metadata is useful but spoofable; keep it as a weak corroborating signal. + if summary.lens_model: + adjustment -= 3 + reasons.append("lens model metadata present") + if summary.software: sw_lower = summary.software.lower() if any(s in sw_lower for s in _SUSPICIOUS_SOFTWARE): adjustment += 10 reasons.append(f"editing software detected: {summary.software}") elif any(s in sw_lower for s in _CAMERA_SOFTWARE): - adjustment -= 2 + adjustment -= 1 reasons.append("camera firmware in Software field") + adjustment = max(-12, min(12, adjustment)) + summary.trust_adjustment = adjustment summary.trust_reason = "; ".join(reasons) if reasons else "no EXIF metadata found" diff --git a/services/general_image_service.py b/services/general_image_service.py new file mode 100644 index 0000000000000000000000000000000000000000..34e1814571a95faa6a9f7e33e9f7d4c6b407ef91 --- /dev/null +++ b/services/general_image_service.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Optional + +import torch +from loguru import logger +from PIL import Image + +from config import settings +from models.model_loader import get_model_loader +from schemas.common import ArtifactIndicator, ExifSummary, VLMBreakdown + +_AI_TOKENS = ("ai", "artificial", "fake", "generated", "synthetic") +_REAL_TOKENS = ("real", "human", "natural", "photo", "authentic") + + +@dataclass +class GeneralImageDetection: + fake_probability: float + label: str + all_scores: dict[str, float] + model_used: str + + +@dataclass +class NoFaceFusion: + fake_probability: float + label: str + method: str + components: dict[str, float] = field(default_factory=dict) + weights: dict[str, float] = field(default_factory=dict) + + +def _fake_probability_from_scores(scores: dict[str, float]) -> float: + ai_scores = [ + p for label, p in scores.items() + if any(token in label.lower() for token in _AI_TOKENS) + ] + if ai_scores: + return float(max(ai_scores)) + + real_scores = [ + p for label, p in scores.items() + if any(token in label.lower() for token in _REAL_TOKENS) + ] + if real_scores: + return float(1.0 - max(real_scores)) + + logger.warning(f"Could not infer AI-generated label from general image labels: {list(scores)}") + return 0.5 + + +def classify_general_image(pil_img: Image.Image) -> Optional[GeneralImageDetection]: + loaded = get_model_loader().load_general_image_model() + if loaded is None: + return None + model, processor = loaded + + inputs = processor(images=pil_img.convert("RGB"), return_tensors="pt") + inputs = {k: v.to(settings.DEVICE) for k, v in inputs.items()} + + with torch.no_grad(): + logits = model(**inputs).logits + probs = torch.softmax(logits, dim=-1)[0] + + id2label: dict[int, str] = getattr(model.config, "id2label", {}) + scores = {id2label.get(i, str(i)): float(p.item()) for i, p in enumerate(probs)} + top_label = max(scores.items(), key=lambda kv: kv[1])[0] if scores else "unknown" + return GeneralImageDetection( + fake_probability=_fake_probability_from_scores(scores), + label=top_label, + all_scores=scores, + model_used=settings.GENERAL_IMAGE_MODEL_ID, + ) + + +def _forensic_fake_probability(artifacts: list[ArtifactIndicator]) -> float: + if not artifacts: + return 0.5 + + weighted: list[tuple[float, float]] = [] + for artifact in artifacts: + weight = 1.0 + if artifact.type == "gan_artifact": + weight = 1.25 + elif artifact.type == "compression": + weight = 0.85 + elif artifact.type in {"facial_boundary", "lighting"}: + weight = 0.60 + weighted.append((weight, float(artifact.confidence))) + + total_weight = sum(w for w, _ in weighted) + if total_weight <= 0: + return 0.5 + return max(0.0, min(1.0, sum(w * score for w, score in weighted) / total_weight)) + + +def _exif_fake_probability(exif: ExifSummary | None) -> float: + if exif is None or exif.trust_adjustment == 0: + return 0.5 + # trust_adjustment is -12..12; positive means more fake, negative means more real. + return max(0.0, min(1.0, 0.5 + (float(exif.trust_adjustment) / 24.0))) + + +def _vlm_fake_probability(vlm: VLMBreakdown | None) -> Optional[float]: + if vlm is None: + return None + scores = [ + vlm.facial_symmetry.score, + vlm.skin_texture.score, + vlm.lighting_consistency.score, + vlm.background_coherence.score, + vlm.anatomy_hands_eyes.score, + vlm.context_objects.score, + ] + authenticity = sum(float(s) for s in scores) / max(len(scores), 1) + return max(0.0, min(1.0, 1.0 - authenticity / 100.0)) + + +def fuse_no_face_evidence( + *, + general_fake_prob: float | None, + artifacts: list[ArtifactIndicator], + exif: ExifSummary | None, + vlm: VLMBreakdown | None = None, +) -> NoFaceFusion: + components = { + "general_detector": 0.5 if general_fake_prob is None else max(0.0, min(1.0, float(general_fake_prob))), + "forensics": _forensic_fake_probability(artifacts), + "exif": _exif_fake_probability(exif), + } + + weights = { + "general_detector": settings.NOFACE_GENERAL_WEIGHT, + "forensics": settings.NOFACE_FORENSICS_WEIGHT, + "exif": settings.NOFACE_EXIF_WEIGHT, + } + + vlm_prob = _vlm_fake_probability(vlm) + if vlm_prob is not None: + components["vlm_consistency"] = vlm_prob + weights["vlm_consistency"] = settings.NOFACE_VLM_WEIGHT + + total_weight = sum(weights.values()) + if total_weight <= 0: + fake_prob = components["general_detector"] + else: + fake_prob = sum(components[k] * weights[k] for k in weights) / total_weight + fake_prob = max(0.0, min(1.0, fake_prob)) + + return NoFaceFusion( + fake_probability=fake_prob, + label="Fake" if fake_prob >= 0.5 else "Real", + method="no_face_general_forensic_fusion", + components=components, + weights=weights, + ) diff --git a/services/image_service.py b/services/image_service.py index e0376764f14b9701345f0152d03b01f81512e2ec..ca87074278e00ab6732e498a0543152c760d171a 100644 --- a/services/image_service.py +++ b/services/image_service.py @@ -4,12 +4,16 @@ import io from dataclasses import dataclass, field from typing import List, Optional, Tuple +import cv2 +import numpy as np import torch from loguru import logger from PIL import Image from config import settings from models.model_loader import get_model_loader +from schemas.common import ArtifactIndicator, ExifSummary, VLMBreakdown +from services.general_image_service import classify_general_image, fuse_no_face_evidence @dataclass @@ -19,6 +23,8 @@ class ImageClassification: all_scores: dict[str, float] models_used: List[str] = field(default_factory=list) ensemble_method: Optional[str] = None + calibrator_applied: bool = False + no_face_analysis: Optional[dict] = None def load_image_from_bytes(data: bytes) -> Image.Image: @@ -55,70 +61,238 @@ def _classify_vit(pil_img: Image.Image) -> Tuple[float, str, dict[str, float]]: return fake_prob, top_label, all_scores -def classify_image(pil_img: Image.Image) -> ImageClassification: - """Run deepfake classification. Uses ensemble (ViT + EfficientNet) when ENSEMBLE_MODE=true, - falls back to ViT-only when EfficientNet is unavailable or ENSEMBLE_MODE=false. +def _crop_face_for_face_model(pil_img: Image.Image) -> Image.Image: + """Best-effort face crop for FFPP-style classifiers trained on face crops.""" + rgb = np.asarray(pil_img.convert("RGB")) + h, w = rgb.shape[:2] + + def crop(x0: int, y0: int, x1: int, y1: int, margin: float = 0.24) -> Image.Image: + bw = max(1, x1 - x0) + bh = max(1, y1 - y0) + pad = int(max(bw, bh) * margin) + x0c = max(0, x0 - pad) + y0c = max(0, y0 - pad) + x1c = min(w, x1 + pad) + y1c = min(h, y1 + pad) + if x1c <= x0c + 8 or y1c <= y0c + 8: + return pil_img + return Image.fromarray(rgb[y0c:y1c, x0c:x1c]) + + try: + from models.model_loader import get_model_loader + + detector = get_model_loader().load_face_detector() + result = detector.process(rgb) if detector is not None else None + if result is not None and getattr(result, "multi_face_landmarks", None): + landmarks = result.multi_face_landmarks[0].landmark + xs = [lm.x * w for lm in landmarks] + ys = [lm.y * h for lm in landmarks] + return crop(int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys))) + except Exception as exc: # noqa: BLE001 + logger.debug(f"FFPP MediaPipe face crop failed: {exc}") + + try: + gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY) + cascade_path = cv2.data.haarcascades + "haarcascade_frontalface_default.xml" + cascade = cv2.CascadeClassifier(cascade_path) + faces = cascade.detectMultiScale(gray, scaleFactor=1.08, minNeighbors=4, minSize=(32, 32)) + if len(faces) > 0: + x, y, fw, fh = max(faces, key=lambda box: box[2] * box[3]) + return crop(int(x), int(y), int(x + fw), int(y + fh)) + except Exception as exc: # noqa: BLE001 + logger.debug(f"FFPP OpenCV face crop failed: {exc}") + + return pil_img + + +def _classify_ffpp(pil_img: Image.Image) -> Optional[Tuple[float, dict[str, float]]]: + """Run the FFPP-fine-tuned ViT (Phase 11.3). Returns (fake_prob, all_scores) or None.""" + loader = get_model_loader() + loaded = loader.load_ffpp_model() + if loaded is None: + return None + model, processor = loaded + + face_img = _crop_face_for_face_model(pil_img) + inputs = processor(images=face_img, return_tensors="pt") + inputs = {k: v.to(settings.DEVICE) for k, v in inputs.items()} + + with torch.no_grad(): + outputs = model(**inputs) + probs = torch.softmax(outputs.logits, dim=-1)[0] + + id2label: dict[int, str] = getattr(model.config, "id2label", {0: "fake", 1: "real"}) + all_scores = {id2label.get(i, str(i)): float(p.item()) for i, p in enumerate(probs)} + fake_prob = next( + (float(v) for k, v in all_scores.items() if k.lower() == "fake"), + float(probs[0].item()), + ) + return fake_prob, all_scores + + +def _has_face_for_routing(pil_img: Image.Image) -> bool: + rgb = np.asarray(pil_img.convert("RGB")) + try: + detector = get_model_loader().load_face_detector() + result = detector.process(rgb) if detector is not None else None + if result is not None and getattr(result, "multi_face_landmarks", None): + return True + except Exception as exc: # noqa: BLE001 + logger.debug(f"MediaPipe face route check failed: {exc}") + + try: + gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY) + cascade_path = cv2.data.haarcascades + "haarcascade_frontalface_default.xml" + cascade = cv2.CascadeClassifier(cascade_path) + faces = cascade.detectMultiScale(gray, scaleFactor=1.08, minNeighbors=4, minSize=(32, 32)) + return len(faces) > 0 + except Exception as exc: # noqa: BLE001 + logger.debug(f"OpenCV face route check failed: {exc}") + return False + + +def _classify_no_face( + pil_img: Image.Image, + *, + artifact_indicators: Optional[list[ArtifactIndicator]] = None, + exif: ExifSummary | None = None, + vlm_breakdown: VLMBreakdown | None = None, +) -> ImageClassification: + general = classify_general_image(pil_img) + fused = fuse_no_face_evidence( + general_fake_prob=general.fake_probability if general else None, + artifacts=artifact_indicators or [], + exif=exif, + vlm=vlm_breakdown, + ) + scores = { + f"no_face_{name}": score + for name, score in fused.components.items() + } + scores.update({ + f"no_face_weight_{name}": weight + for name, weight in fused.weights.items() + }) + if general is not None: + scores.update({f"general_{k}": v for k, v in general.all_scores.items()}) + + analysis = { + "method": fused.method, + "components": fused.components, + "weights": fused.weights, + "general_model": general.model_used if general else None, + "general_label": general.label if general else "unavailable", + "general_unavailable": general is None, + } + models_used = [general.model_used if general else "no-face-forensic-fusion"] + return ImageClassification( + label=fused.label, + confidence=fused.fake_probability, + all_scores=scores, + models_used=models_used, + ensemble_method=fused.method, + no_face_analysis=analysis, + ) + + +def classify_image( + pil_img: Image.Image, + *, + artifact_indicators: Optional[list[ArtifactIndicator]] = None, + exif: ExifSummary | None = None, + vlm_breakdown: VLMBreakdown | None = None, +) -> ImageClassification: + """Run deepfake classification. Weighted ensemble across: + - FFPP-fine-tuned ViT (Phase 11.3, face-trained) — highest weight when present + - EfficientNetAutoAttB4 (face-gated DFDC model) + - Generic ViT (prithivMLmods) + Falls back gracefully when individual models are unavailable. """ + face_present_for_route = _has_face_for_routing(pil_img) + if not face_present_for_route and settings.ENSEMBLE_MODE: + result = _classify_no_face( + pil_img, + artifact_indicators=artifact_indicators, + exif=exif, + vlm_breakdown=vlm_breakdown, + ) + logger.info( + f"Image classify ({result.ensemble_method}) → {result.label} " + f"@ {result.confidence:.3f}" + ) + return result + vit_fake_prob, vit_label, vit_scores = _classify_vit(pil_img) models_used = [settings.IMAGE_MODEL_ID] + scores_out: dict[str, float] = {f"vit_{k}": v for k, v in vit_scores.items()} + + # FFPP inference (may be None if disabled / checkpoint missing). + ffpp_fake_prob: Optional[float] = None + ffpp_res = _classify_ffpp(pil_img) if settings.FFPP_ENABLED and face_present_for_route else None + if ffpp_res is not None: + ffpp_fake_prob, ffpp_scores = ffpp_res + models_used.append("ffpp-vit-local") + scores_out.update({f"ffpp_{k}": v for k, v in ffpp_scores.items()}) if not settings.ENSEMBLE_MODE: - logger.info(f"Image classify (ViT-only) → {vit_label} @ fake_p={vit_fake_prob:.3f}") - label = "Fake" if vit_fake_prob >= 0.5 else "Real" + # ViT-only mode, but still blend FFPP when available — it's strictly better. + if ffpp_fake_prob is not None: + combined = 0.4 * vit_fake_prob + 0.6 * ffpp_fake_prob + method = "ffpp_vit_blend" + else: + combined = vit_fake_prob + method = None + label = "Fake" if combined >= 0.5 else "Real" + logger.info(f"Image classify (ensemble-off) → {label} @ {combined:.3f}") return ImageClassification( - label=label, - confidence=vit_fake_prob, - all_scores=vit_scores, - models_used=models_used, - ensemble_method=None, + label=label, confidence=combined, all_scores=scores_out, + models_used=models_used, ensemble_method=method, ) - # Attempt EfficientNet inference. + # EfficientNet inference (face-gated). loader = get_model_loader() eff_detector = loader.load_efficientnet() - if eff_detector is None: - logger.warning("EfficientNet unavailable — falling back to ViT-only") - label = "Fake" if vit_fake_prob >= 0.5 else "Real" - return ImageClassification( - label=label, - confidence=vit_fake_prob, - all_scores=vit_scores, - models_used=models_used, - ensemble_method=None, - ) - - eff_result = eff_detector.detect_image(pil_img) - if eff_result.get("error") or eff_result.get("score") is None: - # BlazeFace found no face — trust ViT alone. - logger.info(f"EfficientNet no-face fallback → using ViT score {vit_fake_prob:.3f}") - label = "Fake" if vit_fake_prob >= 0.5 else "Real" - return ImageClassification( - label=label, - confidence=vit_fake_prob, - all_scores=vit_scores, - models_used=models_used, - ensemble_method="vit_only_no_face", - ) + eff_fake_prob: Optional[float] = None + face_present = face_present_for_route + if eff_detector is not None: + eff_result = eff_detector.detect_image(pil_img) + if not eff_result.get("error") and eff_result.get("score") is not None: + eff_fake_prob = float(eff_result["score"]) + face_present = True + models_used.append(eff_result["model"]) + scores_out["efficientnet_fake"] = eff_fake_prob + scores_out["efficientnet_real"] = 1.0 - eff_fake_prob + scores_out["efficientnet_calibrator_applied"] = 1.0 if eff_result.get("calibrator_applied") else 0.0 - eff_fake_prob: float = eff_result["score"] - models_used.append(eff_result["model"]) + # Weighted ensemble + if face_present and eff_fake_prob is not None and ffpp_fake_prob is not None: + w_ffpp = settings.FFPP_WEIGHT_FACE + w_vit = settings.VIT_WEIGHT_FACE + w_eff = settings.EFFNET_WEIGHT_FACE + total = w_ffpp + w_vit + w_eff + ensemble_prob = (w_ffpp * ffpp_fake_prob + w_vit * vit_fake_prob + w_eff * eff_fake_prob) / total + method = "weighted_ffpp_vit_eff" + elif face_present and eff_fake_prob is not None: + ensemble_prob = 0.5 * vit_fake_prob + 0.5 * eff_fake_prob + method = "average_vit_eff" + else: + ensemble_prob = vit_fake_prob + method = "vit_only_no_face" - # Simple average ensemble. - ensemble_prob = (vit_fake_prob + eff_fake_prob) / 2.0 label = "Fake" if ensemble_prob >= 0.5 else "Real" logger.info( - f"Image classify (ensemble) → {label} | vit={vit_fake_prob:.3f} eff={eff_fake_prob:.3f} avg={ensemble_prob:.3f}" + f"Image classify ({method}) → {label} | vit={vit_fake_prob:.3f} " + f"ffpp={ffpp_fake_prob if ffpp_fake_prob is not None else 'n/a'} " + f"eff={eff_fake_prob if eff_fake_prob is not None else 'n/a'} " + f"→ {ensemble_prob:.3f}" ) return ImageClassification( label=label, confidence=ensemble_prob, - all_scores={ - **{f"vit_{k}": v for k, v in vit_scores.items()}, - f"efficientnet_fake": eff_fake_prob, - f"efficientnet_real": 1.0 - eff_fake_prob, - }, + all_scores=scores_out, models_used=models_used, - ensemble_method="average", + ensemble_method=method, + calibrator_applied=bool(scores_out.get("efficientnet_calibrator_applied", 0.0)), ) diff --git a/services/job_queue.py b/services/job_queue.py new file mode 100644 index 0000000000000000000000000000000000000000..f79a123062212f25ba921da6c2c4bdb60afb8ccd --- /dev/null +++ b/services/job_queue.py @@ -0,0 +1,80 @@ +"""Phase 19.3 — in-memory async job queue. + +Backed by FastAPI BackgroundTasks for single-worker deployments; the API +surface matches what a future Celery/Redis migration would expose, so +callers don't need to change when we swap the transport. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable + +from loguru import logger + + +@dataclass +class Job: + id: str + stage: str = "queued" + progress: int = 0 # 0..100 + status: str = "queued" # queued | running | done | error + result: Any | None = None + error: str | None = None + created_at: float = field(default_factory=time.time) + updated_at: float = field(default_factory=time.time) + + +class _JobRegistry: + def __init__(self) -> None: + self._jobs: dict[str, Job] = {} + self._lock = threading.Lock() + + def create(self) -> Job: + job = Job(id=uuid.uuid4().hex) + with self._lock: + self._jobs[job.id] = job + return job + + def get(self, job_id: str) -> Job | None: + with self._lock: + return self._jobs.get(job_id) + + def update(self, job_id: str, **fields) -> None: + with self._lock: + j = self._jobs.get(job_id) + if not j: + return + for k, v in fields.items(): + setattr(j, k, v) + j.updated_at = time.time() + + def prune(self, ttl_seconds: int = 3600) -> None: + cutoff = time.time() - ttl_seconds + with self._lock: + dead = [jid for jid, j in self._jobs.items() if j.updated_at < cutoff] + for jid in dead: + self._jobs.pop(jid, None) + + +registry = _JobRegistry() + + +def run_job(job_id: str, stages: list[str], fn: Callable[[Callable[[str, int], None]], Any]) -> None: + """Wrap a callable so it advances stage/progress through `registry`. + + `fn` receives a `progress(stage, pct)` updater it can call. + """ + def progress(stage: str, pct: int) -> None: + registry.update(job_id, stage=stage, progress=max(0, min(100, int(pct))), status="running") + + registry.update(job_id, status="running", stage=stages[0] if stages else "running", progress=1) + try: + result = fn(progress) + registry.update(job_id, status="done", stage="done", progress=100, result=result) + except Exception as e: # noqa: BLE001 + logger.exception(f"Job {job_id} failed") + registry.update(job_id, status="error", error=str(e), progress=100) diff --git a/services/llm_explainer.py b/services/llm_explainer.py index d62e14d239944393e23c208ae480649907fcb0d2..5a31218f60b8e25b0bbc8be0c95b224b9fdc75ba 100644 --- a/services/llm_explainer.py +++ b/services/llm_explainer.py @@ -7,9 +7,11 @@ Results are cached per record_id to avoid re-spending tokens. from __future__ import annotations +import hashlib import json +import threading +import time from abc import ABC, abstractmethod -from functools import lru_cache from typing import Any from loguru import logger @@ -17,8 +19,33 @@ from loguru import logger from config import settings from schemas.common import LLMExplainabilitySummary -# ── In-memory cache keyed by record_id ── +# ── In-memory caches ── +# Keyed by record_id — one row per analysis _cache: dict[str, LLMExplainabilitySummary] = {} +# Keyed by SHA-256 of the prompt — dedups across different analyses of the same content +_hash_cache: dict[str, LLMExplainabilitySummary] = {} + +# ── Circuit breaker: shared by llm_explainer + vlm_breakdown via the helpers below ── +_COOLDOWN_SECONDS = 300 # 5 min after a 429/quota error +_cooldown_until: float = 0.0 + + +def is_rate_limited() -> bool: + """Return True if we're in a post-429 cooldown window. Skip all LLM calls.""" + return time.time() < _cooldown_until + + +def mark_rate_limited(seconds: int = _COOLDOWN_SECONDS) -> None: + """Open the circuit for `seconds`. Safe to call from any LLM caller.""" + global _cooldown_until + _cooldown_until = time.time() + seconds + logger.warning(f"LLM rate-limited — pausing all LLM calls for {seconds}s") + + +def _is_quota_error(exc: Exception) -> bool: + """Heuristic: detect 429 / quota / ResourceExhausted across Gemini + OpenAI SDKs.""" + msg = f"{type(exc).__name__} {exc!s}".lower() + return any(m in msg for m in ("429", "resourceexhausted", "quota", "rate limit", "toomanyrequests")) _PROMPT_TEMPLATE = """\ @@ -48,44 +75,117 @@ Rules: class _LLMProvider(ABC): + name: str = "base" + model: str = "" + @abstractmethod def generate(self, prompt: str) -> str: """Send prompt to LLM and return raw text response.""" + @property + def tag(self) -> str: + return f"{self.name}/{self.model}" + class _GeminiProvider(_LLMProvider): + """Gemini via the new `google-genai` SDK (replaces deprecated `google-generativeai`).""" + name = "gemini" + def __init__(self) -> None: - import google.generativeai as genai - genai.configure(api_key=settings.LLM_API_KEY) - self._model = genai.GenerativeModel(settings.LLM_MODEL) + from google import genai + self._client = genai.Client(api_key=settings.LLM_API_KEY) + self.model = settings.LLM_MODEL def generate(self, prompt: str) -> str: - response = self._model.generate_content(prompt) - return response.text + resp = self._client.models.generate_content(model=self.model, contents=prompt) + return resp.text or "" class _OpenAIProvider(_LLMProvider): + name = "openai" + def __init__(self) -> None: from openai import OpenAI self._client = OpenAI(api_key=settings.LLM_API_KEY) + self.model = settings.LLM_MODEL def generate(self, prompt: str) -> str: response = self._client.chat.completions.create( - model=settings.LLM_MODEL, + model=self.model, messages=[{"role": "user", "content": prompt}], temperature=0.3, max_tokens=300, ) - return response.choices[0].message.content + return response.choices[0].message.content or "" + + +class _GroqProvider(_LLMProvider): + """Groq — free-tier Llama 3.3 70B. Used as failover when the primary hits 429.""" + name = "groq" + + def __init__(self) -> None: + from groq import Groq + self._client = Groq(api_key=settings.GROQ_API_KEY) + self.model = settings.GROQ_MODEL + + def generate(self, prompt: str) -> str: + response = self._client.chat.completions.create( + model=self.model, + messages=[{"role": "user", "content": prompt}], + temperature=0.3, + max_tokens=400, + response_format={"type": "json_object"}, + ) + return response.choices[0].message.content or "" -@lru_cache(maxsize=1) -def _get_provider() -> _LLMProvider: - """Lazy-init the configured LLM provider (singleton).""" - provider_name = settings.LLM_PROVIDER.lower() - if provider_name == "openai": - return _OpenAIProvider() - return _GeminiProvider() +class _ProviderChain: + """Primary provider with optional Groq fallback. On a quota error from the + primary, transparently retries on Groq. The `last_used` attribute tracks + which provider actually produced the response so `model_used` reflects truth. + """ + + def __init__(self, primary: _LLMProvider, fallback: _LLMProvider | None) -> None: + self._primary = primary + self._fallback = fallback + self.last_used: _LLMProvider = primary + + def generate(self, prompt: str) -> str: + try: + text = self._primary.generate(prompt) + self.last_used = self._primary + return text + except Exception as e: + if self._fallback is None: + raise + logger.info(f"{self._primary.tag} failed ({type(e).__name__}) — failing over to {self._fallback.tag}") + text = self._fallback.generate(prompt) + self.last_used = self._fallback + return text + + +_provider_lock = threading.Lock() +_provider_instance: _ProviderChain | None = None # reset to None forces re-init with new fallback logic + + +def _get_provider() -> _ProviderChain: + """Lazy-init the configured provider chain (thread-safe singleton).""" + global _provider_instance + if _provider_instance is not None: + return _provider_instance + with _provider_lock: + if _provider_instance is None: + provider_name = settings.LLM_PROVIDER.lower() + primary: _LLMProvider = _OpenAIProvider() if provider_name == "openai" else _GeminiProvider() + fallback: _LLMProvider | None = None + if settings.GROQ_API_KEY and primary.name != "groq": + try: + fallback = _GroqProvider() + logger.info(f"LLM chain initialized: {primary.tag} → {fallback.tag}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Groq fallback unavailable: {e}") + _provider_instance = _ProviderChain(primary, fallback) + return _provider_instance def _parse_llm_response(raw: str) -> tuple[str, list[str]]: @@ -121,30 +221,22 @@ def generate_llm_summary( Returns: LLMExplainabilitySummary with paragraph, bullets, and model info. """ - # Check cache + # Check record-id cache if record_id and record_id in _cache: logger.debug(f"LLM summary cache hit for record_id={record_id}") cached = _cache[record_id] cached.cached = True return cached + # Circuit breaker — skip the API entirely during cooldown + if is_rate_limited(): + logger.debug("LLM in cooldown — returning fallback summary") + return _fallback_summary(payload, reason="rate_limited") + # Guard: no API key configured if not settings.LLM_API_KEY: logger.warning("LLM_API_KEY not set — using deterministic fallback summary") - - verdict_data = payload.get("verdict", {}) - label = verdict_data.get("label", "Unknown") - score = verdict_data.get("authenticity_score", 50) - - return LLMExplainabilitySummary( - paragraph=f"The DeepShield AI engine has analyzed this media and determined it is '{label}' with an authenticity score of {score}/100. We arrived at this conclusion by passing the file through our deepfake detection algorithms, artifact scanners, and metadata analyzers.", - bullets=[ - f"Overall Authenticity Score: {score}/100", - f"Primary Verdict: {label}", - "Note: Configure an LLM API key for deeper contextual analysis." - ], - model_used="static-fallback", - ) + return _fallback_summary(payload, reason="no_api_key") # Strip heavy base64 fields to reduce token usage slim_payload = {k: v for k, v in payload.items() @@ -155,37 +247,79 @@ def generate_llm_summary( if not k.endswith("_base64")} slim_payload["explainability"] = expl - prompt = _PROMPT_TEMPLATE.format(payload_json=json.dumps(slim_payload, indent=2, default=str)) + prompt_body = json.dumps(slim_payload, indent=2, default=str, sort_keys=True) + prompt = _PROMPT_TEMPLATE.format(payload_json=prompt_body) + + # Content-hash cache — dedups "same analysis re-run" across users / record_ids + content_hash = hashlib.sha256( + f"{settings.LLM_PROVIDER}|{settings.LLM_MODEL}|{prompt_body}".encode("utf-8") + ).hexdigest() + if content_hash in _hash_cache: + logger.debug(f"LLM summary content-hash cache hit sha={content_hash[:12]}") + summary = _hash_cache[content_hash].model_copy(update={"cached": True}) + if record_id: + _cache[record_id] = summary + return summary try: - provider = _get_provider() - raw_response = provider.generate(prompt) + chain = _get_provider() + raw_response = chain.generate(prompt) paragraph, bullets = _parse_llm_response(raw_response) summary = LLMExplainabilitySummary( paragraph=paragraph, bullets=bullets, - model_used=f"{settings.LLM_PROVIDER}/{settings.LLM_MODEL}", + model_used=chain.last_used.tag, ) - # Cache result + # Cache by both record_id and content hash + _hash_cache[content_hash] = summary if record_id: _cache[record_id] = summary - logger.info(f"LLM summary generated via {settings.LLM_PROVIDER}/{settings.LLM_MODEL}") + logger.info(f"LLM summary generated via {chain.last_used.tag}") return summary except json.JSONDecodeError as e: logger.error(f"LLM returned unparseable JSON: {e}") + chain = _get_provider() return LLMExplainabilitySummary( paragraph="Analysis complete. See the detailed indicators below for specifics.", bullets=["LLM explanation could not be parsed"], - model_used=f"{settings.LLM_PROVIDER}/{settings.LLM_MODEL}", + model_used=chain.last_used.tag, ) except Exception as e: + if _is_quota_error(e): + mark_rate_limited() + logger.warning(f"LLM quota hit ({type(e).__name__}) — circuit open for {_COOLDOWN_SECONDS}s") + return _fallback_summary(payload, reason="rate_limited") logger.error(f"LLM explainer failed: {e}") return LLMExplainabilitySummary( paragraph="Analysis complete. See the detailed indicators below for specifics.", bullets=["LLM explanation temporarily unavailable"], model_used="error", ) + + +def _fallback_summary(payload: dict[str, Any], *, reason: str) -> LLMExplainabilitySummary: + """Deterministic summary used when the LLM is unavailable (no key / rate-limited).""" + verdict_data = payload.get("verdict", {}) + label = verdict_data.get("label", "Unknown") + score = verdict_data.get("authenticity_score", 50) + tail = { + "rate_limited": "LLM paused — automatic summary shown during quota cooldown.", + "no_api_key": "Note: Configure an LLM API key for deeper contextual analysis.", + }.get(reason, "LLM explanation unavailable.") + return LLMExplainabilitySummary( + paragraph=( + f"The DeepShield AI engine analyzed this media and determined it is '{label}' " + f"with an authenticity score of {score}/100, derived from deepfake detection, " + f"artifact scanning, and metadata analysis." + ), + bullets=[ + f"Overall Authenticity Score: {score}/100", + f"Primary Verdict: {label}", + tail, + ], + model_used=f"static-fallback:{reason}", + ) diff --git a/services/news_lookup.py b/services/news_lookup.py index 8831afb27b3e5d852cf6c2838c8bd96ceca8420d..439156a9d8ebb5c25aefd414a27bac6c22f0a4e8 100644 --- a/services/news_lookup.py +++ b/services/news_lookup.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import List, Optional, Tuple +from typing import List, Optional from urllib.parse import urlparse import httpx @@ -51,8 +51,10 @@ def _domain_of(url: str) -> str: def _is_factcheck(url: str, title: str) -> bool: - dom = _domain_of(url) - if any(fc in dom for fc in FACTCHECK_DOMAINS): + parsed = urlparse(url or "") + dom = parsed.netloc.lower().replace("www.", "") + path_key = f"{dom}{parsed.path}".lower() + if any(fc in dom or fc in path_key for fc in FACTCHECK_DOMAINS): return True tl = (title or "").lower() return any(kw in tl for kw in ("fact check", "fact-check", "debunked", "false claim", "misleading", "hoax")) @@ -99,11 +101,20 @@ def _compute_truth_override( try: import numpy as np - # Encode input text and all high-trust headlines - source_texts = [ - f"{s.title}" for s in high_trust - ] - all_texts = [input_text[:512]] + source_texts + high_trust = [s for s in high_trust if (s.description or "").strip()] + if not high_trust: + return None + + # Encode input text and high-trust headline+description pairs. Headline + # overlap alone is too weak to override the classifier. + source_texts = [f"{s.title}. {s.description}" for s in high_trust] + input_cmp = input_text[:512] + input_terms = { + t for t in input_cmp.lower().split() + if len(t.strip(".,!?;:()[]{}\"'")) >= 5 + for t in [t.strip(".,!?;:()[]{}\"'")] + } + all_texts = [input_cmp] + source_texts embeddings = st_model.encode(all_texts, convert_to_numpy=True, normalize_embeddings=True) query_vec = embeddings[0] # (D,) @@ -121,7 +132,14 @@ def _compute_truth_override( f"source={best_source.source_name} url={best_source.url}" ) - if best_sim >= _OVERRIDE_SIMILARITY_THRESHOLD: + best_terms = { + t for t in f"{best_source.title} {best_source.description or ''}".lower().split() + if len(t.strip(".,!?;:()[]{}\"'")) >= 5 + for t in [t.strip(".,!?;:()[]{}\"'")] + } + lexical_overlap = len(input_terms & best_terms) / max(len(input_terms), 1) + + if best_sim >= _OVERRIDE_SIMILARITY_THRESHOLD and lexical_overlap >= 0.35: new_fake_prob = min( current_fake_prob * _OVERRIDE_FAKE_PROB_MULTIPLIER, _OVERRIDE_FAKE_PROB_CAP, @@ -153,11 +171,10 @@ def _compute_truth_override( async def _fetch(q: str, country: Optional[str]) -> list[dict]: - target_country = country or "in" - params = {"apikey": settings.NEWS_API_KEY, "q": q, "language": "en", "size": 10, "country": "in"} + params = {"apikey": settings.NEWS_API_KEY, "q": q, "language": "en", "size": 10, "country": country or "in"} try: - async with httpx.AsyncClient(timeout=8.0) as c: + async with httpx.AsyncClient(timeout=httpx.Timeout(5.0, connect=2.0)) as c: r = await c.get(settings.NEWS_API_BASE_URL, params=params) r.raise_for_status() return (r.json() or {}).get("results") or [] @@ -222,6 +239,7 @@ async def search_news_full( source_name=src_name, title=title, url=url, + description=art.get("description") or art.get("content"), published_at=art.get("pubDate"), relevance_score=_relevance(url), )) diff --git a/services/rate_limit.py b/services/rate_limit.py new file mode 100644 index 0000000000000000000000000000000000000000..9d953b13deda313ea3bdf3d65010314e057aee79 --- /dev/null +++ b/services/rate_limit.py @@ -0,0 +1,89 @@ +"""Phase 15.2 — Rate limiting via slowapi. + +Exposes a shared `limiter` plus a request key function that differentiates +authenticated users (keyed by user id) from anonymous clients (keyed by IP). + +slowapi 0.1.9 calls `exempt_when()` with no arguments, so we stash the current +Request in a ContextVar via a middleware (`RateLimitContextMiddleware`) and +read from it inside the exempt predicates. +""" +from __future__ import annotations + +from contextvars import ContextVar + +from fastapi import Request +from slowapi import Limiter +from slowapi.util import get_remote_address +from starlette.middleware.base import BaseHTTPMiddleware + +from services.auth_service import decode_token + +_REQUEST_STASH: ContextVar[Request | None] = ContextVar("_REQUEST_STASH", default=None) + + +def _bearer(request: Request) -> str | None: + auth = request.headers.get("authorization") + if not auth: + return None + parts = auth.split() + if len(parts) != 2 or parts[0].lower() != "bearer": + return None + return parts[1] + + +def _authed_user_id(request: Request) -> str | None: + token = _bearer(request) + if not token: + return None + payload = decode_token(token) + if not payload or "sub" not in payload: + return None + return str(payload["sub"]) + + +def request_key(request: Request) -> str: + """Keyed on user id when authed, IP address otherwise.""" + uid = _authed_user_id(request) + if uid: + return f"user:{uid}" + return f"ip:{get_remote_address(request)}" + + +def is_authed() -> bool: + request = _REQUEST_STASH.get() + if request is None: + return False + return _authed_user_id(request) is not None + + +def is_anon() -> bool: + request = _REQUEST_STASH.get() + if request is None: + return True + return _authed_user_id(request) is None + + +class RateLimitContextMiddleware(BaseHTTPMiddleware): + """Stashes the incoming Request in a ContextVar so slowapi's no-arg + `exempt_when` predicates can read it.""" + + async def dispatch(self, request: Request, call_next): + token = _REQUEST_STASH.set(request) + try: + return await call_next(request) + finally: + _REQUEST_STASH.reset(token) + + +# Per-route rate limits — anon gets strict caps, authed gets generous quotas. +ANON_ANALYZE = "5/hour" +AUTH_ANALYZE = "50/hour" +ANON_REPORT = "2/hour" +AUTH_REPORT = "20/hour" + +limiter = Limiter( + key_func=request_key, + default_limits=[], + headers_enabled=True, + enabled=True, +) diff --git a/services/report_service.py b/services/report_service.py index 0a42b7f4fe3aa983a39cda7d033ffd8ec5d7d259..74b3f33a39f775047fd7fb34ae341200795825e2 100644 --- a/services/report_service.py +++ b/services/report_service.py @@ -2,8 +2,6 @@ from __future__ import annotations import base64 import json -import os -import time import uuid from datetime import datetime, timedelta, timezone from io import BytesIO @@ -111,6 +109,80 @@ def html_to_pdf(html: str, out_path: Path) -> None: logger.warning(f"xhtml2pdf encountered {result.err} warnings/errors during rendering (likely unsupported CSS properties).") +def _fallback_pdf(record: AnalysisRecord, analysis_json: dict, out_path: Path) -> None: + """Generate a minimal report with ReportLab when xhtml2pdf cannot render.""" + from reportlab.lib import colors + from reportlab.lib.pagesizes import A4 + from reportlab.lib.styles import getSampleStyleSheet + from reportlab.lib.units import mm + from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle + + styles = getSampleStyleSheet() + doc = SimpleDocTemplate( + str(out_path), + pagesize=A4, + rightMargin=18 * mm, + leftMargin=18 * mm, + topMargin=16 * mm, + bottomMargin=16 * mm, + ) + verdict = analysis_json.get("verdict", {}) + expl = analysis_json.get("explainability") or {} + summary = analysis_json.get("processing_summary") or {} + + story = [ + Paragraph("DeepShield Analysis Report", styles["Title"]), + Paragraph(f"Record #{record.id} · {analysis_json.get('media_type', record.media_type)}", styles["Normal"]), + Spacer(1, 8), + Paragraph("Verdict", styles["Heading2"]), + Table( + [ + ["Label", verdict.get("label", record.verdict)], + ["Authenticity score", f"{verdict.get('authenticity_score', record.authenticity_score)}/100"], + ["Model label", verdict.get("model_label", "")], + ["Model confidence", f"{float(verdict.get('model_confidence', 0.0)):.3f}"], + ], + colWidths=[45 * mm, 115 * mm], + ), + Spacer(1, 8), + ] + + exif = expl.get("exif") or {} + if exif: + story.extend([ + Paragraph("EXIF Metadata", styles["Heading2"]), + Table( + [[k, str(v)] for k, v in exif.items() if v not in (None, "")], + colWidths=[45 * mm, 115 * mm], + ), + Spacer(1, 8), + ]) + + artifacts = expl.get("artifact_indicators") or [] + if artifacts: + rows = [["Type", "Severity", "Confidence"]] + rows.extend([ + [a.get("type", ""), a.get("severity", ""), f"{float(a.get('confidence', 0.0)):.2f}"] + for a in artifacts[:8] + ]) + table = Table(rows, colWidths=[70 * mm, 45 * mm, 45 * mm]) + table.setStyle(TableStyle([ + ("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#EEF2FF")), + ("GRID", (0, 0), (-1, -1), 0.25, colors.HexColor("#E5E7EB")), + ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"), + ])) + story.extend([Paragraph("Artifact Indicators", styles["Heading2"]), table, Spacer(1, 8)]) + + story.extend([ + Paragraph("Processing Summary", styles["Heading2"]), + Paragraph(f"Model: {summary.get('model_used', '')}", styles["Normal"]), + Paragraph(f"Stages: {', '.join(summary.get('stages_completed') or [])}", styles["Normal"]), + Spacer(1, 10), + Paragraph(analysis_json.get("responsible_ai_notice", "AI-based analysis may not be 100% accurate."), styles["Italic"]), + ]) + doc.build(story) + + def generate_report(record: AnalysisRecord) -> Path: out_dir = _ensure_dir() filename = f"deepshield_{record.id}_{uuid.uuid4().hex[:8]}.pdf" @@ -118,7 +190,11 @@ def generate_report(record: AnalysisRecord) -> Path: data = json.loads(record.result_json) html = render_html(data) - html_to_pdf(html, out_path) + try: + html_to_pdf(html, out_path) + except Exception as exc: # noqa: BLE001 + logger.warning(f"xhtml2pdf failed for report {record.id}, using fallback renderer: {exc}") + _fallback_pdf(record, data, out_path) logger.info(f"Report generated id={record.id} path={out_path} size={out_path.stat().st_size}B") return out_path diff --git a/services/screenshot_service.py b/services/screenshot_service.py index ae5aa3eed6986c0f9a940965bcdd5dfaedfa0dfb..0da8d3e35b236c457f50d8fcc183bbb355ab8377 100644 --- a/services/screenshot_service.py +++ b/services/screenshot_service.py @@ -1,7 +1,7 @@ from __future__ import annotations -from dataclasses import dataclass, field -from typing import List, Tuple +from dataclasses import dataclass +from typing import List import numpy as np from loguru import logger @@ -49,8 +49,10 @@ def run_ocr(pil_img: Image.Image) -> List[OCRBox]: return out -def extract_full_text(boxes: List[OCRBox]) -> str: - return " ".join(b.text for b in boxes if b.text.strip()) +def extract_full_text(boxes: List[OCRBox], min_confidence: float = 0.30) -> str: + filtered = [b for b in boxes if b.text.strip() and b.confidence >= min_confidence] + filtered.sort(key=lambda b: (min(p[1] for p in b.bbox), min(p[0] for p in b.bbox))) + return " ".join(b.text for b in filtered) def map_phrases_to_boxes(boxes: List[OCRBox], manipulation_indicators) -> List[SuspiciousPhrase]: diff --git a/services/storage.py b/services/storage.py new file mode 100644 index 0000000000000000000000000000000000000000..277fb962c0e6bce45c3d8cc67c7474191235c027 --- /dev/null +++ b/services/storage.py @@ -0,0 +1,147 @@ +"""Phase 19.2 — object storage with thumbnails. + +Persists analyzed media under MEDIA_ROOT/{sha[:2]}/{sha}.{ext} so that records +can be rehydrated and re-analyzed without re-uploading. Generates a 400px +thumbnail at MEDIA_ROOT/thumbs/{sha}_400.jpg for history UIs. + +Local-disk implementation only; an S3 adapter can slot in at the same API. +""" + +from __future__ import annotations + +import base64 +import hashlib +import os +from pathlib import Path + +from PIL import Image +from loguru import logger + +MEDIA_ROOT = Path(os.environ.get("MEDIA_ROOT", "./media")).resolve() +THUMB_DIR = MEDIA_ROOT / "thumbs" +THUMB_MAX = 400 + + +def _ensure_dirs() -> None: + MEDIA_ROOT.mkdir(parents=True, exist_ok=True) + THUMB_DIR.mkdir(parents=True, exist_ok=True) + + +def sha256_bytes(data: bytes) -> str: + h = hashlib.sha256() + # Process in 64KB chunks per spec + view = memoryview(data) + for i in range(0, len(view), 65536): + h.update(view[i : i + 65536]) + return h.hexdigest() + + +def sha256_file(path: str | os.PathLike) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while True: + chunk = f.read(65536) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def _media_path_for(sha: str, ext: str) -> Path: + ext = (ext or "").lstrip(".").lower() or "bin" + return MEDIA_ROOT / sha[:2] / f"{sha}.{ext}" + + +def save_bytes(data: bytes, sha: str, ext: str) -> str: + """Write raw bytes to the content-addressed path. Returns relative media path.""" + _ensure_dirs() + dest = _media_path_for(sha, ext) + dest.parent.mkdir(parents=True, exist_ok=True) + if not dest.exists(): + dest.write_bytes(data) + path = dest.relative_to(MEDIA_ROOT.parent) if MEDIA_ROOT.parent in dest.parents else dest + return path.as_posix() + + +def save_file(src_path: str, sha: str, ext: str) -> str: + """Copy an existing file (e.g. temp video) into object storage.""" + _ensure_dirs() + dest = _media_path_for(sha, ext) + dest.parent.mkdir(parents=True, exist_ok=True) + if not dest.exists(): + with open(src_path, "rb") as src, open(dest, "wb") as dst: + while True: + chunk = src.read(65536) + if not chunk: + break + dst.write(chunk) + return dest.as_posix() + + +def make_image_thumbnail(pil: Image.Image, sha: str) -> str | None: + """Write a 400px-max JPEG thumbnail. Returns URL-style path or None on failure.""" + try: + _ensure_dirs() + dest = THUMB_DIR / f"{sha}_400.jpg" + if dest.exists(): + return f"/media/thumbs/{sha}_400.jpg" + im = pil.convert("RGB").copy() + im.thumbnail((THUMB_MAX, THUMB_MAX)) + im.save(dest, "JPEG", quality=82, optimize=True) + return f"/media/thumbs/{sha}_400.jpg" + except Exception as e: # noqa: BLE001 + logger.warning(f"thumbnail generation failed for {sha}: {e}") + return None + + +def make_video_thumbnail(video_path: str, sha: str) -> str | None: + """Grab a frame ~1s in as the video thumbnail.""" + try: + import cv2 # lazy import — heavy + + _ensure_dirs() + dest = THUMB_DIR / f"{sha}_400.jpg" + if dest.exists(): + return f"/media/thumbs/{sha}_400.jpg" + cap = cv2.VideoCapture(video_path) + try: + fps = cap.get(cv2.CAP_PROP_FPS) or 25 + cap.set(cv2.CAP_PROP_POS_FRAMES, int(fps)) + ok, frame = cap.read() + if not ok: + cap.set(cv2.CAP_PROP_POS_FRAMES, 0) + ok, frame = cap.read() + if not ok: + return None + finally: + cap.release() + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + im = Image.fromarray(rgb) + im.thumbnail((THUMB_MAX, THUMB_MAX)) + im.save(dest, "JPEG", quality=82, optimize=True) + return f"/media/thumbs/{sha}_400.jpg" + except Exception as e: # noqa: BLE001 + logger.warning(f"video thumbnail failed for {sha}: {e}") + return None + + +def save_overlay(data_url: str, sha: str, suffix: str) -> str | None: + """Persist a base64 data-URL image as a PNG file for later retrieval. + + Returns a URL-style path like /media/overlays/{sha}_{suffix}.png, or None on failure. + The suffix distinguishes overlay types: 'heatmap', 'ela', 'boxes'. + """ + try: + _ensure_dirs() + overlay_dir = MEDIA_ROOT / "overlays" + overlay_dir.mkdir(parents=True, exist_ok=True) + dest = overlay_dir / f"{sha}_{suffix}.png" + if dest.exists(): + return f"/media/overlays/{sha}_{suffix}.png" + # Strip the data URL prefix (e.g. "data:image/png;base64,") + raw_b64 = data_url.split(",", 1)[1] if "," in data_url else data_url + dest.write_bytes(base64.b64decode(raw_b64)) + return f"/media/overlays/{sha}_{suffix}.png" + except Exception as e: # noqa: BLE001 + logger.warning(f"save_overlay failed for {sha}_{suffix}: {e}") + return None diff --git a/services/text_service.py b/services/text_service.py index 556ac48a8f0195fe5b41d3a789179979e778fae3..ee9f3752e424c642a3f353611cc82d8812bdc768 100644 --- a/services/text_service.py +++ b/services/text_service.py @@ -1,16 +1,16 @@ from __future__ import annotations import re -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import List, Optional from loguru import logger +from config import settings from models.model_loader import get_model_loader FAKE_TOKENS = ("fake", "false", "unreliable", "misinformation") -# --- Sensationalism patterns --- CLICKBAIT_PATTERNS = [ (r"\byou won'?t believe\b", "clickbait"), (r"\bbreaking\s*:", "clickbait"), @@ -22,20 +22,20 @@ CLICKBAIT_PATTERNS = [ (r"\bthis\s+will\s+change\b", "clickbait"), (r"\b(?:everyone|nobody)\s+(?:is|was)\s+talking\b", "clickbait"), ] + EMOTIONAL_WORDS = { "outrage", "shocking", "horrifying", "disgusting", "amazing", "incredible", "unbelievable", "devastating", "terrifying", "explosive", "bombshell", "jaw-dropping", "heartbreaking", "furious", "scandal", "crisis", "chaos", "destroyed", "slammed", "blasted", "exposed", "revealed", } + SUPERLATIVES = { "best", "worst", "greatest", "biggest", "most", "least", "fastest", "deadliest", "largest", "smallest", "ultimate", } -# --- Manipulation indicator patterns --- MANIPULATION_PATTERNS = [ - # Unverified claims (r"\bsources?\s+(?:say|said|claim|report)\b", "unverified_claim", "medium", "Unverified source attribution without specific citation"), (r"\ballegedly\b", "unverified_claim", "low", @@ -46,7 +46,6 @@ MANIPULATION_PATTERNS = [ "Non-specific source attribution"), (r"\brunconfirmed\b", "unverified_claim", "medium", "Explicitly unconfirmed information"), - # Emotional manipulation (r"\boutrage\b", "emotional_manipulation", "medium", "Emotional trigger word designed to provoke reaction"), (r"\bshocking\s+truth\b", "emotional_manipulation", "high", @@ -57,7 +56,6 @@ MANIPULATION_PATTERNS = [ "Conspiracy framing language"), (r"\bopen\s+your\s+eyes\b", "emotional_manipulation", "medium", "Implies audience ignorance"), - # False authority (r"\bexperts?\s+(?:confirm|say|agree|warn)\b", "false_authority", "medium", "Unnamed expert citation without specific attribution"), (r"\bscientists?\s+(?:confirm|prove|say)\b", "false_authority", "medium", @@ -70,7 +68,6 @@ MANIPULATION_PATTERNS = [ "Assertion of fact without evidence"), ] -# NER entity labels to prefer for keyword extraction _NER_PREFERRED = {"PERSON", "ORG", "GPE", "EVENT", "PRODUCT", "NORP"} @@ -84,8 +81,8 @@ class TextClassification: @dataclass class SensationalismResult: - score: int # 0-100 - level: str # Low / Medium / High + score: int + level: str exclamation_count: int caps_word_count: int clickbait_matches: int @@ -95,18 +92,15 @@ class SensationalismResult: @dataclass class ManipulationIndicator: - pattern_type: str # unverified_claim / emotional_manipulation / false_authority + pattern_type: str matched_text: str start_pos: int end_pos: int - severity: str # low / medium / high + severity: str description: str def detect_language(text: str) -> str: - """Detect the primary language of text using langdetect. - Returns ISO 639-1 code (e.g. 'en', 'hi'). Falls back to 'en' on failure. - """ if not text or len(text.strip()) < 10: return "en" try: @@ -115,68 +109,70 @@ def detect_language(text: str) -> str: logger.info(f"Language detected: {lang}") return lang except ImportError: - logger.debug("langdetect not installed — defaulting to 'en'") + logger.debug("langdetect not installed - defaulting to 'en'") return "en" except Exception as e: - logger.debug(f"Language detection failed: {e} — defaulting to 'en'") + logger.debug(f"Language detection failed: {e} - defaulting to 'en'") return "en" -def _scores_to_classification(items) -> TextClassification: - """Convert pipeline output to TextClassification.""" +def _scores_to_classification(items, *, allow_label0_fallback: bool = True) -> TextClassification: + """Convert pipeline output to TextClassification. + + Prefer semantic fake labels. The bundled jy46604790 model uses + LABEL_0=fake/LABEL_1=real, but arbitrary replacement models may not. + """ scores = {i["label"]: float(i["score"]) for i in items} top_label, top_conf = max(scores.items(), key=lambda kv: kv[1]) - # Extract fake probability - fake_prob = 0.0 - if "LABEL_0" in scores: - fake_prob = scores["LABEL_0"] - else: - fake_prob = max( - (p for lbl, p in scores.items() if any(t in lbl.lower() for t in FAKE_TOKENS)), - default=0.0, - ) + + fake_prob = max( + (p for lbl, p in scores.items() if any(t in lbl.lower() for t in FAKE_TOKENS)), + default=None, + ) + if fake_prob is None: + if allow_label0_fallback and "LABEL_0" in scores and "LABEL_1" in scores: + fake_prob = scores["LABEL_0"] + else: + logger.warning(f"Could not infer fake label from text model labels: {list(scores)}") + top_label = "uncertain_label_mapping" + top_conf = 0.0 + fake_prob = 0.5 + return TextClassification(top_label, top_conf, fake_prob, scores) def classify_text(text: str, language: Optional[str] = None) -> TextClassification: - """Classify text as fake/real. - Routes to multilingual model when language is non-English and the model is configured. - """ text = (text or "").strip() if not text: return TextClassification("unknown", 0.0, 0.0, {}) loader = get_model_loader() + is_non_english = bool(language and language != "en") + if is_non_english and not settings.TEXT_MULTILANG_MODEL_ID: + logger.warning(f"No multilingual text model configured for language={language}; returning uncertain score") + return TextClassification("unsupported_language", 0.0, 0.5, {}) - if language and language != "en": - pipe = loader.load_multilang_text_model() - else: - pipe = loader.load_text_model() + pipe = loader.load_multilang_text_model() if is_non_english else loader.load_text_model() out = pipe(text[:2000], truncation=True, top_k=None) items = out[0] if isinstance(out[0], list) else out - clf = _scores_to_classification(items) + clf = _scores_to_classification(items, allow_label0_fallback=not is_non_english) logger.info( - f"Text classify [{language or 'en'}] → {clf.label} @ {clf.confidence:.3f} " + f"Text classify [{language or 'en'}] -> {clf.label} @ {clf.confidence:.3f} " f"fake_p={clf.fake_prob:.3f}" ) return clf def score_sensationalism(text: str) -> SensationalismResult: - """Compute a 0-100 sensationalism score from structural/linguistic signals.""" if not text: return SensationalismResult(0, "Low", 0, 0, 0, 0, 0) words = text.split() total_words = max(len(words), 1) - excl = text.count("!") caps = sum(1 for w in words if w.isupper() and len(w) > 2) - clickbait = sum( - 1 for pat, _ in CLICKBAIT_PATTERNS - if re.search(pat, text, re.IGNORECASE) - ) + clickbait = sum(1 for pat, _ in CLICKBAIT_PATTERNS if re.search(pat, text, re.IGNORECASE)) emotional = sum(1 for w in words if w.lower().strip(".,!?;:") in EMOTIONAL_WORDS) superlative = sum(1 for w in words if w.lower().strip(".,!?;:") in SUPERLATIVES) @@ -190,12 +186,11 @@ def score_sensationalism(text: str) -> SensationalismResult: score = int(min(100, max(0, raw))) level = "Low" if score < 30 else ("Medium" if score < 60 else "High") - logger.info(f"Sensationalism → {score} ({level}) excl={excl} caps={caps} cb={clickbait} emo={emotional}") + logger.info(f"Sensationalism -> {score} ({level}) excl={excl} caps={caps} cb={clickbait} emo={emotional}") return SensationalismResult(score, level, excl, caps, clickbait, emotional, superlative) def detect_manipulation_indicators(text: str) -> List[ManipulationIndicator]: - """Scan text for manipulation linguistic patterns with positions.""" if not text: return [] indicators: List[ManipulationIndicator] = [] @@ -210,28 +205,20 @@ def detect_manipulation_indicators(text: str) -> List[ManipulationIndicator]: description=description, )) indicators.sort(key=lambda i: i.start_pos) - logger.info(f"Manipulation indicators → {len(indicators)} found") + logger.info(f"Manipulation indicators -> {len(indicators)} found") return indicators def extract_entities(text: str, max_k: int = 6) -> List[str]: - """Extract keywords via spaCy NER (PERSON, ORG, GPE, EVENT preferred). - Falls back to frequency-based extraction when spaCy is unavailable or text is too short. - """ if not text or len(text.strip()) < 20: return _extract_keywords_freq(text, max_k) - loader = get_model_loader() - nlp = loader.load_spacy_nlp() - + nlp = get_model_loader().load_spacy_nlp() if nlp is None: - # spaCy not available — use frequency fallback return _extract_keywords_freq(text, max_k) try: - doc = nlp(text[:5000]) # cap for performance - - # Collect named entities, preferring high-value types + doc = nlp(text[:5000]) preferred: List[str] = [] other: List[str] = [] seen: set[str] = set() @@ -248,28 +235,24 @@ def extract_entities(text: str, max_k: int = 6) -> List[str]: other.append(norm) entities = preferred + other - if len(entities) >= 2: logger.info(f"NER extracted {len(entities)} entities: {entities[:max_k]}") return entities[:max_k] - # Not enough entities — supplement with frequency keywords freq_kws = _extract_keywords_freq(text, max_k) combined = entities + [k for k in freq_kws if k.lower() not in seen] return combined[:max_k] - except Exception as e: - logger.warning(f"spaCy NER failed: {e} — falling back to frequency extraction") + logger.warning(f"spaCy NER failed: {e} - falling back to frequency extraction") return _extract_keywords_freq(text, max_k) def _extract_keywords_freq(text: str, max_k: int = 6) -> List[str]: - """Frequency-based keyword extraction (original implementation, kept as fallback).""" stop = { - "the","a","an","is","are","was","were","be","been","being","to","of","and","or","but", - "in","on","at","for","with","by","from","as","that","this","it","its","has","have","had", - "will","would","can","could","should","may","might","do","does","did","not","no","so", - "than","then","there","their","they","them","we","our","you","your","he","she","his","her", + "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", "to", "of", "and", "or", "but", + "in", "on", "at", "for", "with", "by", "from", "as", "that", "this", "it", "its", "has", "have", "had", + "will", "would", "can", "could", "should", "may", "might", "do", "does", "did", "not", "no", "so", + "than", "then", "there", "their", "they", "them", "we", "our", "you", "your", "he", "she", "his", "her", } words = re.findall(r"[A-Za-z][A-Za-z\-']{2,}", text or "") freq: dict[str, int] = {} @@ -281,5 +264,4 @@ def _extract_keywords_freq(text: str, max_k: int = 6) -> List[str]: return [w for w, _ in sorted(freq.items(), key=lambda kv: (-kv[1], kv[0]))[:max_k]] -# Back-compat alias: routes that still call extract_keywords get NER-first behaviour extract_keywords = extract_entities diff --git a/services/video_service.py b/services/video_service.py index b930978391f7376a2dcb0ec7f0da7178743be367..b1beaec6ce766fdecf57c7f1fd86021174df9d23 100644 --- a/services/video_service.py +++ b/services/video_service.py @@ -11,6 +11,7 @@ from PIL import Image from config import settings from models.model_loader import get_model_loader from services.image_service import _classify_vit +from services.video_temporal import TemporalAnalysis, compute_temporal_score @dataclass @@ -38,6 +39,9 @@ class VideoAggregation: frames: List[FrameAnalysis] = field(default_factory=list) models_used: List[str] = field(default_factory=list) face_detector_used: str = "mediapipe" + calibrator_applied: bool = False + # Phase 17.1 — temporal consistency + temporal: Optional[TemporalAnalysis] = None FAKE_TOKENS = ("fake", "deepfake", "manipulated", "ai", "generated", "synthetic") @@ -86,25 +90,38 @@ MIN_FACE_FRAMES = 3 def _has_face_mediapipe(pil: Image.Image) -> bool: detector = get_model_loader().load_face_detector() + if detector is None: + return False arr = np.array(pil) res = detector.process(arr) return bool(getattr(res, "multi_face_landmarks", None)) +def _score_efficientnet_face(eff, face) -> float: + face_t = eff._face_tensor(face) + import torch + with torch.inference_mode(): + logit = eff.net(face_t.unsqueeze(0).to(eff.device)) + from scipy.special import expit + raw_prob = float(expit(logit.cpu().numpy().item())) + return float(eff._calibrate(raw_prob)) + + def _analyze_with_efficientnet( frames: List[Tuple[int, float, np.ndarray, Image.Image]], -) -> Tuple[List[FrameAnalysis], str, List[str]]: +) -> Tuple[List[FrameAnalysis], str, List[str], bool]: """Primary path: use EfficientNet + BlazeFace per-frame. Returns (frame_results, detector_used, models_used).""" loader = get_model_loader() eff = loader.load_efficientnet() if eff is None: logger.warning("EfficientNet unavailable — falling back to ViT video pipeline") - return _analyze_with_vit(frames), "mediapipe", [settings.IMAGE_MODEL_ID] + return _analyze_with_vit(frames), "mediapipe", [settings.IMAGE_MODEL_ID], False results: List[FrameAnalysis] = [] face_detector_used = "blazeface" models_used = [f"{settings.EFFICIENTNET_MODEL}_{settings.EFFICIENTNET_TRAIN_DB}"] + calibrator_applied = bool(getattr(eff, "calibrator_applied", False)) for idx, ts, frame_bgr, pil in frames: # Pass RGB to EfficientNet (process_image expects RGB array). @@ -114,21 +131,17 @@ def _analyze_with_efficientnet( has_face = bool(faces) if not has_face: - # Fallback: check MediaPipe so we don't silently miss faces. - has_face = _has_face_mediapipe(pil) - if has_face: - face_detector_used = "blazeface+mediapipe_fallback" + fallback_crop = eff._fallback_face_crop(frame_rgb) + if fallback_crop is not None: + faces = [fallback_crop] + has_face = True + face_detector_used = "blazeface+crop_fallback" fake_prob = 0.0 label = "unknown" if has_face and faces: - # Run EfficientNet on the best face from BlazeFace. - face_t = eff._face_tensor(faces[0]) - import torch - with torch.inference_mode(): - logit = eff.net(face_t.unsqueeze(0).to(eff.device)) - from scipy.special import expit - fake_prob = float(expit(logit.cpu().numpy().item())) + # Run EfficientNet on the best face/crop and apply the same calibration as image inference. + fake_prob = _score_efficientnet_face(eff, faces[0]) label = "Fake" if fake_prob > 0.5 else "Real" elif not has_face: label = "no_face" @@ -142,11 +155,11 @@ def _analyze_with_efficientnet( suspicious_prob=fake_prob, is_suspicious=(fake_prob >= 0.5) and has_face, has_face=has_face, - scored=has_face, + scored=has_face and faces, ) ) - return results, face_detector_used, models_used + return results, face_detector_used, models_used, calibrator_applied def _analyze_with_vit( @@ -176,6 +189,7 @@ def aggregate( frame_results: List[FrameAnalysis], models_used: Optional[List[str]] = None, face_detector_used: str = "mediapipe", + calibrator_applied: bool = False, ) -> VideoAggregation: if not frame_results: return VideoAggregation(0, 0, 0, 0.0, 0.0, 0.0, True) @@ -206,6 +220,7 @@ def aggregate( frames=frame_results, models_used=models_used or [settings.IMAGE_MODEL_ID], face_detector_used=face_detector_used, + calibrator_applied=calibrator_applied, ) @@ -213,10 +228,26 @@ def analyze_video(video_path: str, num_frames: int = 16) -> VideoAggregation: frames = extract_frames(video_path, num_frames=num_frames) if settings.ENSEMBLE_MODE: - frame_results, face_detector_used, models_used = _analyze_with_efficientnet(frames) + frame_results, face_detector_used, models_used, calibrator_applied = _analyze_with_efficientnet(frames) else: frame_results = _analyze_with_vit(frames) face_detector_used = "mediapipe" models_used = [settings.IMAGE_MODEL_ID] + calibrator_applied = False + + agg = aggregate( + frame_results, + models_used=models_used, + face_detector_used=face_detector_used, + calibrator_applied=calibrator_applied, + ) + + # Phase 17.1 — temporal consistency on BGR frames + try: + bgr_frames = [f[2] for f in frames] + timestamps = [f[1] for f in frames] + agg.temporal = compute_temporal_score(bgr_frames, timestamps) + except Exception as exc: # noqa: BLE001 + logger.warning(f"Temporal analysis failed: {exc}") - return aggregate(frame_results, models_used=models_used, face_detector_used=face_detector_used) + return agg diff --git a/services/video_temporal.py b/services/video_temporal.py new file mode 100644 index 0000000000000000000000000000000000000000..44ac19da5b60fde6528cab40a207812edd5c39f9 --- /dev/null +++ b/services/video_temporal.py @@ -0,0 +1,233 @@ +"""Phase 17.1 — Temporal Consistency Module. + +Analyses optical flow variance, luminance flicker, and blink timing across +sampled video frames to produce a temporal_score (0–100, higher = more natural). +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Tuple + +import cv2 +import numpy as np +from loguru import logger + + +@dataclass +class TemporalAnalysis: + temporal_score: float # 0–100, higher = more natural / authentic + optical_flow_variance: float # mean inter-frame flow-magnitude variance + flicker_score: float # 0–100 (high = suspicious micro-flicker) + blink_rate_anomaly: bool # True when blink timing is unnatural + blink_intervals: List[float] = field(default_factory=list) + diagnostics: dict = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Optical-flow variance +# --------------------------------------------------------------------------- + +def _compute_optical_flow_variance(frames_bgr: List[np.ndarray]) -> float: + """Mean variance of inter-frame optical-flow magnitudes. + + Real videos show consistent, smooth motion; deepfake temporal inconsistencies + appear as irregular per-frame flow jumps. + """ + if len(frames_bgr) < 2: + return 0.0 + + flow_mags: List[float] = [] + for i in range(len(frames_bgr) - 1): + prev_gray = cv2.cvtColor(frames_bgr[i], cv2.COLOR_BGR2GRAY) + curr_gray = cv2.cvtColor(frames_bgr[i + 1], cv2.COLOR_BGR2GRAY) + + h, w = prev_gray.shape + scale = min(1.0, 320.0 / max(h, w, 1)) + if scale < 1.0: + dsize = (max(1, int(w * scale)), max(1, int(h * scale))) + prev_gray = cv2.resize(prev_gray, dsize) + curr_gray = cv2.resize(curr_gray, dsize) + + flow = cv2.calcOpticalFlowFarneback( + prev_gray, curr_gray, None, + pyr_scale=0.5, levels=3, winsize=15, + iterations=3, poly_n=5, poly_sigma=1.2, flags=0, + ) + mag, _ = cv2.cartToPolar(flow[..., 0], flow[..., 1]) + flow_mags.append(float(np.mean(mag))) + + return float(np.var(flow_mags)) if flow_mags else 0.0 + + +# --------------------------------------------------------------------------- +# Luminance flicker +# --------------------------------------------------------------------------- + +def _compute_flicker_score(frames_bgr: List[np.ndarray]) -> float: + """Flicker score 0–100 derived from inter-frame luminance variance. + + Deepfake GAN generators introduce subtle luminance micro-flicker that + manifests as high variance in the difference sequence. + """ + if len(frames_bgr) < 2: + return 0.0 + + mean_lums = [ + float(np.mean(cv2.cvtColor(f, cv2.COLOR_BGR2GRAY))) + for f in frames_bgr + ] + diffs = [abs(mean_lums[i + 1] - mean_lums[i]) for i in range(len(mean_lums) - 1)] + if not diffs: + return 0.0 + + mean_diff = float(np.mean(diffs)) + std_diff = float(np.std(diffs)) + flicker_ratio = std_diff / (mean_diff + 1e-6) + return float(min(100.0, flicker_ratio * 50.0)) + + +# --------------------------------------------------------------------------- +# Blink-rate anomaly (FaceMesh EAR) +# --------------------------------------------------------------------------- + +def _compute_blink_anomaly( + frames_bgr: List[np.ndarray], + timestamps: List[float], +) -> Tuple[bool, List[float]]: + """Detect unnatural blink timing using MediaPipe FaceMesh eye-aspect-ratio. + + Returns (anomaly_detected, blink_interval_list_seconds). + Natural blink rate: ~15–20/min → intervals ~3–4 s. + Anomalies: perfectly regular cadence (std < 0.05 s) or rate > 2/s. + """ + try: + import mediapipe as mp + mp_face_mesh = mp.solutions.face_mesh + except ImportError: + return False, [] + + # Landmark indices for left eye (vertical & horizontal pairs) + EYE_V = (159, 145) + EYE_H = (33, 133) + BLINK_THRESH = 0.25 + + ear_seq: List[Tuple[float, float]] = [] + + with mp_face_mesh.FaceMesh( + static_image_mode=True, + max_num_faces=1, + refine_landmarks=True, + min_detection_confidence=0.5, + ) as mesh: + for frame_bgr, ts in zip(frames_bgr, timestamps): + rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + res = mesh.process(rgb) + if not (res and res.multi_face_landmarks): + continue + lm = res.multi_face_landmarks[0].landmark + h, w = frame_bgr.shape[:2] + + def pt(idx: int) -> np.ndarray: + return np.array([lm[idx].x * w, lm[idx].y * h]) + + v = float(np.linalg.norm(pt(EYE_V[0]) - pt(EYE_V[1]))) + h_dist = float(np.linalg.norm(pt(EYE_H[0]) - pt(EYE_H[1]))) + ear = v / (h_dist + 1e-6) + ear_seq.append((ts, ear)) + + if len(ear_seq) < 3: + return False, [] + + blink_times: List[float] = [] + in_blink = False + for ts, ear in ear_seq: + if ear < BLINK_THRESH and not in_blink: + blink_times.append(ts) + in_blink = True + elif ear >= BLINK_THRESH: + in_blink = False + + if len(blink_times) < 2: + return False, [] + + intervals = [ + round(blink_times[i + 1] - blink_times[i], 3) + for i in range(len(blink_times) - 1) + ] + mean_iv = float(np.mean(intervals)) + std_iv = float(np.std(intervals)) + anomaly = (std_iv < 0.05 and len(intervals) > 2) or mean_iv < 0.5 + return bool(anomaly), intervals + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def compute_temporal_score( + frames_bgr: List[np.ndarray], + timestamps: List[float], +) -> TemporalAnalysis: + """Compute temporal consistency for a list of BGR video frames. + + Args: + frames_bgr: BGR numpy arrays in temporal order. + timestamps: Corresponding timestamps in seconds. + + Returns: + TemporalAnalysis with temporal_score 0–100 (higher = more authentic). + """ + if len(frames_bgr) < 2: + return TemporalAnalysis( + temporal_score=50.0, + optical_flow_variance=0.0, + flicker_score=0.0, + blink_rate_anomaly=False, + diagnostics={"frames_analyzed": len(frames_bgr)}, + ) + + flow_var = 0.0 + try: + flow_var = _compute_optical_flow_variance(frames_bgr) + except Exception as exc: # noqa: BLE001 + logger.warning(f"Optical flow failed: {exc}") + + flicker = 0.0 + try: + flicker = _compute_flicker_score(frames_bgr) + except Exception as exc: # noqa: BLE001 + logger.warning(f"Flicker score failed: {exc}") + + blink_anomaly, blink_intervals = False, [] + try: + blink_anomaly, blink_intervals = _compute_blink_anomaly(frames_bgr, timestamps) + except Exception as exc: # noqa: BLE001 + logger.warning(f"Blink rate analysis failed: {exc}") + + # Score composition + # flow_var: real ~0–2; deepfake inconsistencies push higher → penalise + flow_auth = max(0.0, 100.0 - min(100.0, flow_var * 15.0)) + flicker_auth = 100.0 - flicker + blink_penalty = 20.0 if blink_anomaly else 0.0 + + # Weights: 50% flow, 40% flicker, 10% blink + raw = 0.50 * flow_auth + 0.40 * flicker_auth + 0.10 * (100.0 - blink_penalty) + temporal_score = float(max(0.0, min(100.0, raw))) + + logger.info( + f"Temporal: flow_var={flow_var:.4f} flicker={flicker:.1f} " + f"blink_anomaly={blink_anomaly} → temporal_score={temporal_score:.1f}" + ) + + return TemporalAnalysis( + temporal_score=round(temporal_score, 2), + optical_flow_variance=round(flow_var, 4), + flicker_score=round(flicker, 2), + blink_rate_anomaly=blink_anomaly, + blink_intervals=blink_intervals, + diagnostics={ + "flow_component": round(flow_auth, 1), + "flicker_component": round(flicker_auth, 1), + "frames_analyzed": len(frames_bgr), + }, + ) diff --git a/services/vlm_breakdown.py b/services/vlm_breakdown.py index 50ab212b81579d5eeeb475d0191d603fa52809ca..354d2c51ac36be6d208309dc67400e4246fdbf9d 100644 --- a/services/vlm_breakdown.py +++ b/services/vlm_breakdown.py @@ -14,6 +14,7 @@ from PIL import Image from config import settings from schemas.common import VLMBreakdown, VLMComponentScore +from services.llm_explainer import is_rate_limited, mark_rate_limited, _is_quota_error _cache: dict[str, VLMBreakdown] = {} @@ -81,6 +82,11 @@ def generate_vlm_breakdown( logger.debug("LLM_API_KEY not set — skipping VLM breakdown") return None + # Shared circuit breaker with llm_explainer — skip during cooldown + if is_rate_limited(): + logger.debug("VLM in cooldown — skipping") + return None + provider = settings.LLM_PROVIDER.lower() model_id = settings.LLM_MODEL @@ -101,16 +107,19 @@ def generate_vlm_breakdown( logger.error(f"VLM breakdown: unparseable JSON from LLM: {e}") return None except Exception as e: + if _is_quota_error(e): + mark_rate_limited() + logger.warning(f"VLM quota hit ({type(e).__name__}) — circuit open") + return None logger.error(f"VLM breakdown failed: {e}") return None def _call_gemini(image: Image.Image, model_id: str) -> VLMBreakdown: - import google.generativeai as genai # type: ignore - genai.configure(api_key=settings.LLM_API_KEY) - model = genai.GenerativeModel(model_id) - response = model.generate_content([_PROMPT, image]) - return _build_breakdown(_parse_response(response.text)) + from google import genai + client = genai.Client(api_key=settings.LLM_API_KEY) + response = client.models.generate_content(model=model_id, contents=[_PROMPT, image]) + return _build_breakdown(_parse_response(response.text or "")) def _call_openai(image: Image.Image, model_id: str) -> VLMBreakdown: diff --git a/temp_reports/deepshield_43_262befa5.pdf b/temp_reports/deepshield_43_262befa5.pdf new file mode 100644 index 0000000000000000000000000000000000000000..70559f14acb75462bd1c9772936b8739fec1d964 --- /dev/null +++ b/temp_reports/deepshield_43_262befa5.pdf @@ -0,0 +1,161 @@ +%PDF-1.4 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R /F3 4 0 R /F4 7 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/BaseFont /ZapfDingbats /Name /F3 /Subtype /Type1 /Type /Font +>> +endobj +5 0 obj +<< +/BitsPerComponent 8 /ColorSpace /DeviceRGB /Filter [ /ASCII85Decode /FlateDecode ] /Height 229 /Length 5359 /SMask 6 0 R + /Subtype /Image /Type /XObject /Width 229 +>> +stream +Gb"/lH&PbR*QMH)hlU(7C$Sr<(Sa]7>@UrR2nSj?Bc<0n2@['2)Ir)"UJ7kmg8#*4lnYjr<<3+4`FZ-l+p\6=&?<#BDSQP%J8`!A=^KFbBBJ`O);`#op[7M*Hg^WlT7l,Vh##r@Id_L+q;$*a,ldoFzzzzzzzzzzz!!!!q%EUuZMN[d5M?Ei_b3iWi"l\!_llrtnH-lA6Y?eT=(0d@FlR%J?rlu>8cCb%QTClL`puUkibBW?tl>&LH-kq(E72;E_q`tdH+_.$`+ng6jLI":h,*[]DhZPsMICWJ/_WTgnB3`^$T$E'2;D"bQWW#qlkiMBS.Zl@ue^o&;[TFtpUkQLTG,4%CKh?iefJoA8oNO6L_ER$#W3/9NV.)[9#oMrZ.VMItKOS^I?[)gFH7pXm:PKf2Lda.NM@X!eVC5ktc^"l['"blOYu:P^t@=BO1@ZS(BZ%<=:,6gT8j!M;S"f?Z6s");H'%j3b7ihl9R?NRLM5brbtRd8Nh''Q6ZS;ngUd94QP\dT\Rk6]p:\dH<][?AEdu55*n^it@VFf5df6_9@a%>%c3VWp&*Ps770fig=sZV:F+Lk:(*KUicYdQn5W7(Of6583be=6!IPBksC:XKigF.NB<>-<85!Q#X\f1;h:0(pefoG8^L8H.H+\Pie"EauEd&]M6W^;Q^BM+$2`#<^WX+3\KMn*C?*"5PlR,E.CRIq'<>7SLsPrD_IseHG%`ARe$<](<[Qbc1;QF@]IM)[H3OPn01b;((!td5bK@Ch&TZM/=mY[Amh:naoklFO./k[=VX=pqocA^MUd/!k*YF4!pD[TuEbC3V&]H(GI,e@oskf/;1QLn\`8&rPNZ+MX,"4jC]Zd'YA`L0Y$MPlY^T(%u5)=$6+(55RCTBW,&5YhZrtmWq7/Dc/fAl?;gV0bK@aQ)=I%<$/<_U2Sl'IKbI3]qS0N`HU(gYKrSX!M=kB=?Bi,Inb[ncAUpfkMaT"4Dg.&U85&U.a('_0j(9ZL.<_k8oA,]sld.YPPI4JhZ/K"4dr3;7a&ikM(3dB8f$B[gl8"9')%jA%DP5DhTYXP*p5_8J&qUV[c_3ng/dkDe?J'jC5B.:3;p/2[=(o]G(sXekKPX2*5X$]FKceqp<&3=UeFaB8Lb0F?8N,-6A-]lLVKHY]1NrB,,7:2s`&0@c[&gg&7(Z[R@6n;d.2);]]IsWCnlOXIF%`;ADo]o+um.Mn;ut\,EQ:J#cmM93>*Z8S=J$m&/NQ:;]\RYN87X8"D%d4rMi)<(r[)"7*s-SnXYto]2:n[]9_tlIF;c%&s)]N7N1>NkbLN-!3%'Z#F5=3Mp(*]oFq\C+t?mqGan:Cj'pNh:F=[hNk3ihMkC;8i6,]'@r09rN*(=aMLHU*cC)GYD[QWFQ%ANDQFd1@s.EKl`5N?P?S"b'tj_m\F%n@@G!Fq$d5iu(=mMl"]s7,WjjGP'3>(NY-Dm1W`/5\V_#>=p,'>SSb.rEWah!qZA9(%Sm_BodnK-O5Je^2>r(L'??+9aZ:\Rs2+Yd/?]]/gF,nqJcM59@f]uhCU@9'Oh0%QD_*-4,@&$W%"d\YO]NNd^qS7#Wk+-:pVhS*&.qetr*b[2N"r%2gD>eW&BBob81pdFc]rbjKtr5WM9k9_lgPq^RT.I-qG]]OHkPo&3_1b#_ROYi/(a<%p4q:i]pE)+Top=8W_aXI`7sBr/LI>PZ2"hb'ZEh".uAgJ'),S_Ka&qU$(.nqRpjN]l4lN@#R4q11l;hG(tk_^8_YTIYNga2A@h]j=/[?W6p,`.?'USf\O-RP,?]DC8Iu[M0:VO/`/lJ:?9QD=5r!nSqnfja'4mMZJ6$qa0p/"*Zh[86SiX!i)Gp/7u9EA5!CV(7P1d'Z[EN,&]@Pri$bH5?hXC;ACdf4;a2J"K'ZQs0e#E(L0,L9]T%$+e[fT6gZoY.N9*H5Fr9+1e>XLJUUZTD^3iY'S"E3/:Z>e$Y'gQ/U7i:454]F[dQ'WT\m.QAD:i+?&'GXc!1$t^su\3:4_dO*B:@7LIB`]AGZScpR5dbXB(;l4UJ05G>IN=9&/>rf\0aIcR7))ca_!gN@#J.1Jg/\k-lc(J`\1fOL'A=ab=IGQZV\aWE0G*p*(7kAnLkGQR\(3Y3B4:0^,(ahjR$0.t@Q]30g-aUcT-bG1*dWgp`84=8(Pi31m-NaS'sVn"(A]YgFiI(b9g59)DD.A'p@\@JB,;_On?(W#f&cV^aoIhen%=4I`#%`n:LMkDSdQWWu"cPdM1`WtV'%6Bk[&]-0WidCEFV&a?=I$_"<]]7FcK$koW.(8sWUH.I?GOg,FF)Y4%8icDS_'-sXs:Nm2[WRU*6TS7%1PCG,Q5CA!6S4HIe?#LB'rP;%R/#dObfS.hGFh9`bG;B;uQK"q(2a*ul\KWP3R*9hl8lX6[=\pSu^V67BTf^r.o8cl\b`T`\n\gda>R'$WOiET8ja.+XN4#/\[;VI5'!8CK[\2,L/M+Vl!f`T/!Q;>7F4+/c.ajhbBEnt(/V"QKo5*:OWeoWlo8[.Q4/0n?'I9O49&8kMM#,3>O/#.BOgp+``,$*uajJbf'S1AG.#=I)FS*!W7Wa`h*ddM/J=bElF/(0U27%!t&m-7-Xfsigi)#MCndW=iNi>4kc3Y$01p/D93$O3=*&(>nZ?0>i+"\1"?XUu^.9#`\RDRF=*(N,44#@$hcGu3#nLM3KsK'kG%3U>orP^hnGH9^A8sOIEND$--X@kd44@uHEKt-o[Z%+bK2/WYn$&[iMM^N<4`@0:51V9k>+#WJT3C&c(T[f)[Bl"9fedmI;.[8Y#4[9^+"-XPWVlq?(-bIC1"^KMqRH/*PXd%"oS;mZrKCEHGS8eRpWU4si7gK([$krj]u;.qo^D,[2NqlZ5pr7U)S7u]*D?5"P5@nHIfY7.n0YA-Id\X)#\hpRAnl_g2q-G,pq*P0T82.<6A1:-%F$[U#Sq/b+=m>U0P,\Q6;:K6.`*qP?4g[_gRs%7QA?:.`X68>Ws48epWUF57`P5cBfnn9DiNp72f2_gm+%uIKZs%NVU#;oZ;u7MeWd\>J@P51;W*f5d3EL^3ric/:op(b'X#)*O9*&+@76fR5gs$m&_e*ZVpF>:+99Pg\Cd8a-_SOf(2XF7%]!hT9?%^*+d\MTn[",4-d1=@4HCpbXRbp]OVaOu=;]Vu__OnGs>1YddX+*&[_k4G+]af"hG4aM)N2nCM'cgM[4StYa0XIk-["bUJ?bLf?"gTV"<8c^stL11j8L_n`_O\Imq%@elAsrY3pX.N&J+]2&gU//@pi\/.'Wp95MDo"qpUEXCYm:D1'mFq&,u/\Cto/Z+JRGGiUaI6n(M8Aid?:eX4_'[VoOec=pBqNS:Tc:Rt$UW#LMmZ(d6N3nX'T)k_!cOYAAgE0V'lc<\KAn^ZBu*KD79c)roV+U&Zd?PqDu(LMQ6WBY7Pc3,1uZ%t#Do/la]fCh;6B!2YJ(TZ6DgF?p)%GA\"iG?H[[Ha3o2JgjT$YPfVdQfLkiZFG`bW+L7N?M<%d04B,(Ou7)CrA&G[pf/9W,_t<((-oLJD7]da^.7Z.OJ-N6Ul!/OSJe'C6,%RUE?YM8pRaM]B<)B'?-\QhL!iIb6]RNDmW-G.RBo-F$O$/\lo\O3CM+=?/Fie/Jo^b/1),QSGZ6SJpWIC19KjE94"CTcl4aod,"@K*K^TfBLsn+W0B72O+pRBQNP=;2c^q^#oPKCFUieO=BA)/`e42!_Wendstream +endobj +6 0 obj +<< +/BitsPerComponent 8 /ColorSpace /DeviceGray /Decode [ 0 1 ] /Filter [ /ASCII85Decode /FlateDecode ] /Height 229 /Length 3925 + /Subtype /Image /Type /XObject /Width 229 +>> +stream +Gb"/lH#OKe*lmHWS3S20!oXh!LU+^ITI^[[5Y3PCD2_A>"H\/7!hL^TJd;aX6jRfM2JaQ[6HQ&T#ZtOba0(p>!cfllE6,1*5RnD>?KAfaRH].g,nSon^RjkbJZ\qLV2n<"q`EW_pE<*0JNt=h#Qt3[5TgUD"9O->+:ne]!Wa!>I%0i#\g=h$Q.Wn;W%c9C.WsaF2*4U#,(,l[1Y_NX*^.#3N7F\0n]o+"f0/A"mJO6k:KedCt<-ek__VK(t9-@T*MOW<<)c`;f(D4-o70.(Csb^WT$'UX9a;%],/(t$Fi8.uU4bU/[V!"IdOd@;0(U90G:&Gm+k*>n%1$3J)pbD/JfD*X]tj"=`QY:pqO/tJ,4[>FC[QsH\+'GeA1cG+M/g.5ngY)nWS;h&=p?!")O"K+\[8s<'LD6E,4[@oi(^0SW''lVi+5AXr>.:;G0=%I*8I8InhG>hG=kAcKm\-[bAs)lrSfuWCbgRMFXP,I1Xc%@(XToCuU."qc*2X!oUBo#hGf"&`0<[VLk_`X-fcK:hOQ?mqQP&E,,,*rbCcD#05`]kJ,.*>1`<@4tdH.3cVpaF]+O"HEuYQ$tb*,B2ZtnEf-PaL9T^5TogLMN)%JI2Wdr/`l2FXVh(d[nn5kX(D?F3QW=q@&!M:_+P=CTIcD@G@8&d+kbsFRR.s<>hf'FJqZCO`YYmRbg&SIU*[8(]akUBi1;`;Y3@9qhB!u+a#[M2s'3Ufp72?&0aW[3ANQiZ;,8.i8rG53R-$dV42VhJRaS)W#P$npVG#._!,hcg^%1(u-Ae$,d'gX?Kh*s=c&[R@f(j@_d@->AtOHr6q3+;gemLmcQim0tc"^"c-:#?GckJLRdmUP2$b6Y=#g<$(1@*aH5c=!=.jWLr/N(u3A,%t0a/q#?.)S558I]Z&?MY--2i8XYh2Mt.Y`K!jh!#n3_M3ia,.H`M7CK(C`Vokc.oP/XgRZpfG)PojE:)mIK7!ag9^ELIts;keLg!]r.m?M,Xmss[2dSZ(qQnu+?KJg/d9e/%,o_E8I=dTNr++]LGF?^o;]m;N/HV?GdF/gSJi-lhNhSf-(dC]l1(*S%)&2)(aFdH"b0+lN90f76Bl"am@/r\:Xelal";/2gDp+oh(_VB-F&LV-+Tnp@,.YL;]1hVo!j:CD99[6qMk"i`,ddKP.dO_uT7*>m,e&86*PkGLQ/4adVJLDtsg\g_4](:RdY)!c+GEVFaMEq?5ICZtgA%[^io+_4jROBs%ImP96QjC>KFh6Ue.INA;?UeaQ:-Ye9e^I'B1VH5KY/d42EV2p'Nk!AhL'Z6)n!5[]T3e=BNpr\".9O_k)DBrg]QFg#F0o9Zo3p'k-W<2>YV3D!4aAO?@YnY8=ciT]+ZjYd#JRkqS3Bo1f-V1eGmAAEZ%CCpe)oKNk!5N/.dqNrf+Ym0sM+:It$Xdc9kh"O\#O/0?0eh]"E+-8?raZ2Z3"j6-%2l\H-13N:+jm4[;M/D];jg-/ji9^GY).7HW@6[G=D\>WQ4LZZqhc@EL7dA^LYam2M3*6Y^/0Hr-iZubQMCsT.;XZA^kU)/IkU?)pF,&F5_etR+UuGjqeSdY);)qj?9lEPH,W[MML+F]!+HbT_FG%`.1a(68%,$2>$L=HnVip;F301FP&=9/SAJ^'^)(o6icP%@.Ko04[)(CL]7=*%[HC=`u6n2=Rgq@C'k]uC]DELsbmO-<(*att0?3Y[nclI4bC\_K;>[e0b@QHQ+=[@hr\1963*U:]CVD"Us^NZKLG5IcC.R#p%^>Wk^-Mqk$:B+ejOcG]2Q'q9:7>uHf-0%"7:0)g9le1-gK$I9HT^2ts3`[/2g[2b?kc<%)dM+Z%++u\,+)jsM?]42ghF>1E#606orp*`N":7HWHiPc$rpF&UQ>Ksge8DV,K9O8q0F%ip=iG+shqeU7e*#eYdf<57(2Ih_pER)-I@`m(cWG&-O2tD'g#cGubiZ=fKS!h)]@bf$ji'el)[]?gI$m_-JR#`B(TVL%@9TXmgWk2nPj6q&fD[g!BE$ZLFIX-bT6d'c:.8dIqTQ]bd&2gUbqZtGC_5@)(7,U07(=^r!<9lTQIlH)Q=CYEe6QIIDKq&bXu=#H]0F'):r@T*Xpa:]b=?ch\'iWipn9LXoW/?g!L9VKrT3mendstream +endobj +7 0 obj +<< +/BaseFont /Symbol /Name /F4 /Subtype /Type1 /Type /Font +>> +endobj +8 0 obj +<< +/Contents 19 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 18 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /XObject << +/FormXob.41509ef0052ebe76076294b9a047666f 5 0 R +>> +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +9 0 obj +<< +/Contents 20 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 18 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +10 0 obj +<< +/Outlines 12 0 R /PageMode /UseNone /Pages 18 0 R /Type /Catalog +>> +endobj +11 0 obj +<< +/Author () /CreationDate (D:20260426220945+05'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260426220945+05'00') /Producer (xhtml2pdf ) + /Subject () /Title (DeepShield Analysis Report \204 04e253be-e02e-4c92-b942-47b23f85bf1a) /Trapped /False +>> +endobj +12 0 obj +<< +/Count 2 /First 13 0 R /Last 13 0 R /Type /Outlines +>> +endobj +13 0 obj +<< +/Count -4 /Dest [ 8 0 R /Fit ] /First 14 0 R /Last 17 0 R /Parent 12 0 R /Title (Verdict) +>> +endobj +14 0 obj +<< +/Dest [ 8 0 R /Fit ] /Next 15 0 R /Parent 13 0 R /Title (Verdict) +>> +endobj +15 0 obj +<< +/Dest [ 8 0 R /Fit ] /Next 16 0 R /Parent 13 0 R /Prev 14 0 R /Title (EXIF Metadata) +>> +endobj +16 0 obj +<< +/Dest [ 8 0 R /Fit ] /Next 17 0 R /Parent 13 0 R /Prev 15 0 R /Title (Artifact Indicators) +>> +endobj +17 0 obj +<< +/Dest [ 8 0 R /Fit ] /Parent 13 0 R /Prev 16 0 R /Title (Processing Summary) +>> +endobj +18 0 obj +<< +/Count 2 /Kids [ 8 0 R 9 0 R ] /Type /Pages +>> +endobj +19 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 2614 +>> +stream +Gb!;fD0)1;')q<+0jhAHC`VP]BCL>lX^c:Xii@=+gY/\!Pm^/.S"'H^O<*5^^Rc#[KS>OqjCFHH5YY&#dlmXf!u)@B9ATW3"JL=(d,>:"PbHJ5;1SlhI6o#F#@A"ei?,23-,kOW`DF>U%_*3,(*4.`357YUV%>#.2)8i$S&D#tZPl@n:]Q&TU6n>mlPgDg)D*Wd!%gL,imEq$+ifOU%R9+>@YejPWMZcb+0_>\QF?=5V'Y8`s#2HhHs0B$IH\AKP_"#797p5p=]ZqnqLDGZ$`0:sFV[,/Ns2b,A:b.(%k[4UOcjdu?[R%$V"tL7?#:eq-h/6TfLK"L-=6@"7i22e_pVm^YS-WNZ@QD4"-E?e;8Bn:uS2,po-B)qki85T3?/%rfm+"6JlP+U4O"0S]pZK%EKagS9,T=PIDm&:+=c]aZ)o(fnNTe;sgcC'r*FBb?!=Ai.me["^:mNBpm,jWVT&CS@"/],h:;8hMj4Z\MLGd3dn^`H++J5I_]bkaTu,sqlc`h&G(0?gQ?j7<0Th*lT-Vq4"\%(>J7!EW#*M-eenn0ibLep%TN"F+TfE@'7MM5LZ7RRAOW>RbB?q),R$VYHg9WV4?gW9YLm:H@]d.P;:)SO6.4;;hPgEbMaIssLGB;UWb;&Sa[jU,Rg*7CWf$)Y-[=":=3>Tj1_%H5BJ8g^_k0E$>K+_)TDNl6pBL"=9sVA>*#QS/qi,A=gePrh&>KO;hV8KBEd!fmD5=HC?jU?Upkj/a?fp,9`1ZS$8RSJoNk8b7l4SA5uVbU.d-_,9dN+s:K6c4IF,A.X;r.pP`TXdoo,G;ec)L/m&'J#ffj(/;QZ,r+Z%5.5Aq?OnD&"IY]iRJ5@:[0mjIs/R--FOP-8bLe'eh:0$ql4@1]ABiY/.g8jR2+;)k[IS`P`j9>VU5pa>:nf_gMu?[M.L;?B:!2K2;m1;>N.VVERIQnG2!ZkL&]13q[!LkbkIk-TYc8L6p@@HT8-GMAVX.bCba=7>-^Jt?7pM6%(;bZb,;8ua"?ku9Y:o_.J,C?H:f26i/`eM,#tXBf^%TL4nW49PG`?r;E>S$,J@l$Hs%S$/<7kN*;c1a8=)U0]Sj+>Yscm8GP(f99s.4`W"2U29M&hHaCCT[WZ84IjEtfs59[LR0U9`NjmDq/iBc8pa,*(5k/oP!s8G@Q9+3=;_*O>;uF_AX+Y.d-62j"9MZN%M`0hB(!nQeP)C]T;j!TKdVcZ&+8iBJs1g&i'."D-li=qj[AMftt<%O47rB$6nCL!2Uoq/uDle?E:b52?!n]_ijRm`+T,Et^_oh^?q`?u!ZlV/o@,p861=6"+b:8<38V-/0?is=sc:m^0Lp#9K1`Ataa@*a"9=d=W^`Q>.V>+H9^NL.c4?N,,fQJp,"@@4n6Wu.LAWFLCASH0qU$8e$pId0ang'[Es"i-IGHOClcf0%P&l@3M4GYTAs/+C]C.^;HHP*P$07mX/2CgbBZ<6PaihMNt!G$\F9(7!.hRsQkXhjK/7EMH=&uQ?.LesV#UNtAC)gTgVc\_cip,2("17CO8=>rAaBE50-CTP)Wk=(;_M=L^"ds)'?ef,W*3bu1ZtD3J13.nU%`_q-$ci9O9FJ2d87e/%*S53qYU)O!'38*Igi\CU8\\2qPdVA<6gfLFBMEfE;!j)MgqE%7GkiYKieXbEUjZUnpUBm=r"<[j/u^iMe/&fCNEhT?r]k$.1s8)opdi%QbMrqjL4fq4o:ISe)J`'@HF+Ls#'e\X/&ndh_O)oDYp0lGj.,/kn-fiL/9Y4"`&P!K8#VpX_iPk]nNUulCc]M:l0MJ=qmn^.qbb:)K\`#o9VTMVp$2FW(IBqLk`mEQZYu^c/QCWefp2A6k$YR19f5!,5PT?c2u~>endstream +endobj +20 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 432 +>> +stream +Gat=e:J\hm&B4*cMK^d@P:erGG%nW5$6_,NBYA[?AYE+;EOQ)Hp'ucnc=XC=fg#(t\,-G7#'I3'!Y6.Q'*9no'durlZ=c:2^$jf_-=(N0Ubmj]0f2@R@2ql+eM/4$k8cF=EO\D&dpG$LFDR%U6);`(a^DtV:0j,.)OacKQnfQ$M.%R;2>WBbmbZ+/DO,OTIb3)`>&8"/Rc1)WX2:l@A=qZc3`I2r=g5u\>p?KDc-]T6g\cl=BsFlbX`':;`ST^Y<>`?fqk,?SnV'?\Q>#+2[o2O]]dp1NP#^NnI$qUH?CuQf]0h'_CO9MA_4")kUOWfpgkR?8Z`t.F]>h7_5J,&EVlJhh8qF"+^>+XJp0-G\o+8lje;+qrCYkJUOu1R\5c$M:&sg`URhW&)1$[,d`J%,=`YMDi.cr!=Y;^^l#jD3g,>s[)~>endstream +endobj +xref +0 21 +0000000000 65535 f +0000000061 00000 n +0000000122 00000 n +0000000229 00000 n +0000000341 00000 n +0000000424 00000 n +0000005987 00000 n +0000010120 00000 n +0000010197 00000 n +0000010465 00000 n +0000010670 00000 n +0000010757 00000 n +0000011078 00000 n +0000011152 00000 n +0000011264 00000 n +0000011352 00000 n +0000011459 00000 n +0000011572 00000 n +0000011671 00000 n +0000011737 00000 n +0000014443 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 11 0 R +/Root 10 0 R +/Size 21 +>> +startxref +14966 +%%EOF diff --git a/temp_reports/deepshield_43_7e094c15.pdf b/temp_reports/deepshield_43_7e094c15.pdf new file mode 100644 index 0000000000000000000000000000000000000000..cee10b22c83b2906eaa91d0a6c7103b0be47560e --- /dev/null +++ b/temp_reports/deepshield_43_7e094c15.pdf @@ -0,0 +1,161 @@ +%PDF-1.4 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R /F3 4 0 R /F4 7 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/BaseFont /ZapfDingbats /Name /F3 /Subtype /Type1 /Type /Font +>> +endobj +5 0 obj +<< +/BitsPerComponent 8 /ColorSpace /DeviceRGB /Filter [ /ASCII85Decode /FlateDecode ] /Height 229 /Length 5146 /SMask 6 0 R + /Subtype /Image /Type /XObject /Width 229 +>> +stream +Gb"/lH&PbR*QM#shlU(7Bre2?Ftq]-[PcX4G-d*VR&2TPA(+9/VD5M31tpI5W_L(K%(hJ"8l\,A6ua(p1*?i(71cK)$#VFA0E]V4lT+/&Re&0'(hS':]Q\kghK-H(S3J>]%.Cq)iA\MmcL:[+4q@Vnzzzzzzzzzzzz8jP3XiKE\ee04j_W:tLZ_tsL$7Nq#[$k@[PF\qs=[I%,)@B+Y@mrIsco%BYW\_e-_?S3D59X``Hcf[GUE*V:?IFn-tchIA\#f9CD.A]oI]nO@s/s$E3$2m<>76=SNFj4sS/TUR8@_s"W@7[@PlcOT=n$l\*c7LbkNpF3Hk@ojNe-V@C^"4`<7YpT4%URP/l[$q4S;t-XlDYc9bpE[Ej0ucU(Z-R.LXjGnG]`)3Ah!Z?sORj7l#"Vs=@tr;aj7:K15,r?O,f/8eT]$Abs:FLS,H_G-)\,rk?po7Z4W?9'e)S.B%uAa[jg(KR$FYVe\+O%o]]CCf,=k(=PKd?I<jK6T/]k4oO7p7uTIE]Urob2&aI!7)57jP*1`A2fE<^FR@5]5nG6*APH3XO7c'Xj'&H:R)ApUEL#C'hj+]QFbfD?SVO:o/9dJ.u]GiI731LARb"_k#Jc`.K8-u]a7A`u59^2spl\.Nn\o",XFd..XXk"6,egFh\\PeAM`k6j?crG!.^t;gqmG+.j\WnbX_Uu[@"1Yt^UiXblQSDdXE\>pQX57>$Y*e`joAX=Zf::l0c`;(Wa[GU&g9,45(]h3KK!&B^nd"d/AR!dMPbMk.Oq,T!^%8mM36]$URFeik7+&@U$18+]CD.@V2t%]-m2f-jACG9=&L&N5?=@0KWl@DuP4Pi?91Gq@&(Agh[/1G+3T%K5IE/irABP7)YI3#m30qTLi2uR&0U_Gh8dXL%p=)B#aCTiVDe?3CaMho;r5*:h1:g;7j/j^PKM5dXNB1n:Z]4[P@rnp_Y+![;)lDq?g3kObpNofB6:g>dCgMk@=oGnTr.]JcUG:m6[GlWJSgdP6O]KQaU^%dplI).')lB%4Zsec(4@6dWCNB02.Q,'*-V=ns5;_eC1kH-S*)P!.-@E\CX,gRV*LdC1[M/L'&/GmVJ71>i+(hji$qpR`_aamNStn8/):R;=9LS1N`f9A2VeoHMYKW;<9cgH]9+s"UWYR)Nm@'Sqq&'REjR+aAAO(7Z]RmfA1qMBs0lT:>b?W`J'atWohGU9XCTZ4W\d-??im[;[SF@)a[,(/f?Ir,kiB&QIhJ6W0d"d8:*6WH%^#ReJCQ)as0Z'/0aCf0D9E1V`Ib49YbhHmYKphorq]_ICUD%@iI&4s3M*fGZtW%'.Yrnk#0&E(C!Zf]dt#sXV%g>^I(KL*__R^ep;LIa@$Mp/Xr5s!J1n8^^DES$Jqu`:fQ=Nr];AoO@_3PjBkZ#p&+D4a=HgIVa1f]kX'BUr*Ud+Di&j,;8,>hMLf$YuAHM>7S:t^DCNqgPR1$s_"=sFI75[RpE#Lie[gT2TlnmM1!)1TVNpjiD3BHTF=WET[.eFdO./tu.^qPoos67Gj:=7mST'>J#>I$;lNP_p_LK]fs1(qZp[pnTSjW(ii%HUh.W8+uFkaKi\H$#<,XT]Xh6a^>`LTV/;M=%":ZEA5').#>L9a@@_CZPO&:TaU4K&NKLFmW:ckh9+B$E^@19Wn']ZQ(SgZ*$K8iJehgDO;L>6,J'SX,F-+/pPsQd]7.l#PJBB\o*rUGJ4c`Y&)*Ef$fEDEnGVeZSLNT_jcCYA^)/L*5s>rs[iFk7OSXZ[T#W2WFIFk%jHDILml/f@6t^q3S"\tnR/Yj?,$p\]s+KO$?r6e>t"@\ec`]l[h1WGFEol+P?G:m';LTW<>(CITALG$Wd-.bjcc+LS>,FA%?_F#e>Z*bP7O;ZQ)gbRoSrX$f-k/Q4j[H7(.MqJe<;2h>JZh#u/m9CH?4#KE=)[2\'T\G:h-[&1;U7\>%8Smp,Jmh[gTjZ@N6$k''=H9u%81`a\KlpIDbqLF]T\!7]2EZU))d3U%'!LEQ(69?gMMBLlYeI??+MFjGVKT#Xm=?@/I,1O/g:Y2O\7]jC\.Hg4=qr!l)4UDc-'jt5WYbSM#T5K+XP+GZO@]JaMm4CBPcb0bL5sUf>ot<'Y'I>b&G>u#'AC^)h*p^]f'F*#$jsrNGhf__8g$OM.V^;FqM9sm4d+mX4Ed]W2T/894u*BN4k=Cp9Ycc^fA^JWtqmGonD0a@e=Z[N1G7qjkA-uCud'AMmeS"eq4"3oaYKC2@L0rcD/'kV/\XVg1E4J%WF,/g/Tb-$065b0uob?G1=oS9k3g6-_D%H.g,YhKq3dRl)KDArb%G+(5@4rS(+i%q$UG)!oW[9>%OB?]5gI=@a[9VN>:JK;/\T=ohWj\PMb-d"8A]<=3?DEW68G3ph@QHB1@Jtq&@H+Lhg2r*UY!MDf:(3lDXicMH0?oYRk>]q,kBru))m],]d.D-:)q74?_kWodRquf[5hHp4@g6N(td,9q6e$[?R?r5nJ?ki-IS"B9g%:a$gn#nDS8/O*L.fPG-Oa-_u;,:Pg)o%4JQ&Ye&-MiG8%Hgo<6c>2'p_D\;0DGE4*9jK@s=IlDcnS)*.SafMtu9>2A=WY[:)9O@e'E02GOnM:mIEeKFSrp:i+)SB_^M5?`ih)@[!SCGC"aC,,FXG]fR#;*lb)N=WO&+(N?^*1sD+r[UBI>SZ;:f"RITN,;oP`h+0-XqQS1hNaDZ!E\Sm"1)2a3I`pV\ckF'rSP=VYqb]INPX"bOIYt7VQ\%ZG+S@;6=C?-AE)S`O`VjMU<*HZ-:g9V2LCVt/*6FQs))ZL;!'!c(:XeBl3PGu!h#q9;:X.8BomM]9LQS-1aGQ_Gi]%5L:.B0bj0Xq9=\b,:6K)lYVnDU5!K!bpY=CQqDMG?5[oc]F%^R2#l;,[[;&+7A2S3A_<%/Tb8K!O::f>_:#qae'M"(cMpabb-6L05rJ2q'0A+q3T%r')7Jl,-sKCLUi`qD[;DV)Edhq4+8^@"b-47,o6Ij7Z:+jAe>e#j]Of+Z12KX`;P2=I]B.5N';LVr]*pd40*%`D*W6R"TeCrN\)#216/N3WTP>QJpeZU]pJ7iuM'[bR`>"g?R_>&fQJoE(Z1k?a[hE]lIF(UN@%3J)!QXkb+B4%r%T0S"XmTSbbk?QIa7Fp[.G\/CQ@`H"qT4Q&L&T-(Mel\1*s5`(`/(S%MOFaUSW;rPFF7IT$C6:D-jl^`#pF^]E*^4b$O_uq>8c`D)N*a&4au4>HI47ol.'%8KTooDRq$5t4oaGZ$`!4!k67G!pd#aM63uCN]@YA[U4A6//"ae!J=-B;zzzzzzzzzzzzq$Qm]_lePJ~>endstream +endobj +6 0 obj +<< +/BitsPerComponent 8 /ColorSpace /DeviceGray /Decode [ 0 1 ] /Filter [ /ASCII85Decode /FlateDecode ] /Height 229 /Length 3807 + /Subtype /Image /Type /XObject /Width 229 +>> +stream +Gb"/lH#OKe*ln/k>TA<.Ltp(2ZG;'b/.l,QTpE)E7]ANP8C1?HcOsYPKS<5l#g6XYfH;;Gae^No=CN`"Y+,h#E\480aK?[+8OuBsOnY_c]!j!eEkhGCFSF";eXq=>?=iubhs$^rIbjsB,)>ukY4WbJm#,\t#bOJHTF_,F.1b$@mcLAN#lStgeCj>*]%EHT%b@M2`pfYu1$IQ?%XQ\+[fmri4r)6j``jqW@^c`OborLoGY@28QEFtM]MNK#QD*F5G>FrG16Gq(-G'2N>Fe@!2)>#f>54b^.iOm!&`(u,QHk33L\linf@+heS]erG>rJMaqd"Z_=;7lOLHNON-Amf@DVE.C5gDW]<[leiXQSYH9QP^dCB]M+`ZN7#4Derq>C>OrA?,fZ_9%fj3ah1F(t:S8>6h@I!_tE-Ns+7`M@ff[FP.Q(kOshaUY&p?Ag>NJ.&b,mAJi@@E,i"OC/nDk,2C8&kUmN^u3+RD/n&[0-`k+']Ujr&K:;'iao-X,S)[Oc*<3pRDn]bJId'tcl;``O=HI:JV7B&Wa&9Lbt0Y?6pn=^JC"VV.FJH!\m$.L[(Sm^_L-Ms\0.[LE\eB5,8kjs!7>B.-;u]G63u)ZTc*S4CgcGAlOZYNF\[2=K<)qiJeTCC<<0J-J&jhanLfNro\Cf]+[Tj%fI5T"@WLDOJ/b#="E=E&I`BDI#r1XH"`A`@$$+jE#J\i+;,)At5&bTBL_u%CJBa*\FIsD&#bebd^p;IU$F/3e"SbabLQOP;KGJ-]OBk&fTGkfGHs\3q3apXUhPp$PZ);cXqdF[<`r_2gg]5(9+\4.iJJH`Xc#7AOb`oOO\MT!dcg[G##HVboDMggqFV^cNs,,QopF>NPN^;ZM!b@`(\5,B_!SK2fb#gIlS:D'VIa)YA,3\VG,EN=A]pCX0f?S$%QWYHobaeD9`&cShCZ[\A?)Ac*Mu\hC1sSB_(PiYqA0!ah!52n=+J4p]`0j*-2DQBqR/R_jO[UD]:'-SIIu&CJ+eOmL&0mP&*I7;>5Vd;McNqJ$'3KeG0cO"t%5+PO,M..0`.N*WQH)BhL]mi_K,decVe$sr!u'6+b:_0L:TR-fD$g%K&Kp!0!9lk2@t94NS?F&UBDJlH\lh2,PK4ear<:QX"r74PqAZP>,gC4CnRMtP$ZGTl>RL8mNG#BBl%LXE/D)fXcd/GX/cBCBJ>n?j_'MM\^XodjE^h8asIR8\.-M0^G(.m;31c$6u3EF\J+X6)iMR7E;rtQ``nqLR2!.K6(*NR6n^V/`cSo"D@fP2eQHqX>%/VM\BqD"T?4GA=XYJQGPDg9\\5+QQ$;"Q=^%tr"'EESfaV.PkE>?a8sEFn6OBLKlA#&$(fOg.\6fQ3oTpLWf^o8rWdTNMQ=3Dh:GM%be>6"peL!SP>"@:?s>Q/3W/!^pd.)dct(>^!_'PR?sN9TebFj7H\_YT)T()_$9-\Kd8-,\SoE+P4&s/-;s0nK"P*'%2PC!S`W_D*bU8lY&6dulnbAj@<@oLVR`@mM&MDXq[mh[2,[78Ecnn/ih@*neMJk>fkU4u+%e\;b,>YFQ.71R.ecUFi8BD^kg5#*oZa@d+1]=6(%"6ZWA!1r0p3EM12Z_sD-=6s'PNm*'BXX3cM!DaMd(r'l@IE6*'@.H?LLbf)7IKpMV,`,0EK$2aj]U/`9(Mr`XK;GJU47Z(!"l::,h%mG+`as">4I"$+FQuHj"T7YBbk(l(.lVK/:7-UIM3E)[Y6AL**.Yas^K._jq,$QK("4EA\(EMq`,^sE#?6bF)rYdJG03nMZnb/F8^2Gd`2tPW#jKDKO8g;5%aV&1a:7,L'fHmd^1e!?"I/!:d60-$BdLujhtnm0BCDqAb;+9Ph^D>l#bSfPrRCK.q1TME$H%A',SY#7Q\Xp+.V6=@>tQ#3T]b4Kbs0$>U)S_>F`@4aOi*r-8-%h<6$6sRO0]MNs-cPqck++pKL*3O@\7I3_]'b/:>jUb6i$ZaX/JXA'[<@C.-.,>bXeA8;q1P\upgBG5Gjr0Za>SHB.9D9pTsN]&jqZrUrE'*C"&KS-FL02nOEPH520FM6F6FpLt=FKYLD9[a&?3R]Osb*&Mq"+ASf2/Sp/.P%Jbm1&V=e/EXKF75D,HF7%b+endstream +endobj +7 0 obj +<< +/BaseFont /Symbol /Name /F4 /Subtype /Type1 /Type /Font +>> +endobj +8 0 obj +<< +/Contents 19 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 18 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /XObject << +/FormXob.6f7cec236f627fd3da88975beb700761 5 0 R +>> +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +9 0 obj +<< +/Contents 20 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 18 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +10 0 obj +<< +/Outlines 12 0 R /PageMode /UseNone /Pages 18 0 R /Type /Catalog +>> +endobj +11 0 obj +<< +/Author () /CreationDate (D:20260426221430+05'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260426221430+05'00') /Producer (xhtml2pdf ) + /Subject () /Title (DeepShield Analysis Report \204 04e253be-e02e-4c92-b942-47b23f85bf1a) /Trapped /False +>> +endobj +12 0 obj +<< +/Count 2 /First 13 0 R /Last 13 0 R /Type /Outlines +>> +endobj +13 0 obj +<< +/Count -4 /Dest [ 8 0 R /Fit ] /First 14 0 R /Last 17 0 R /Parent 12 0 R /Title (Verdict) +>> +endobj +14 0 obj +<< +/Dest [ 8 0 R /Fit ] /Next 15 0 R /Parent 13 0 R /Title (Verdict) +>> +endobj +15 0 obj +<< +/Dest [ 8 0 R /Fit ] /Next 16 0 R /Parent 13 0 R /Prev 14 0 R /Title (EXIF Metadata) +>> +endobj +16 0 obj +<< +/Dest [ 8 0 R /Fit ] /Next 17 0 R /Parent 13 0 R /Prev 15 0 R /Title (Artifact Indicators) +>> +endobj +17 0 obj +<< +/Dest [ 8 0 R /Fit ] /Parent 13 0 R /Prev 16 0 R /Title (Processing Summary) +>> +endobj +18 0 obj +<< +/Count 2 /Kids [ 8 0 R 9 0 R ] /Type /Pages +>> +endobj +19 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 2590 +>> +stream +Gb!;f=`<%c&q8H9W0[OL_5c9L]=h".ELb,n*otZX#$4k='hOghicotOU3VTY`CR7D-_gp'.TRA#.b;RR;BS5T/,i0:$Q$eU'KXDp36&-ST"f[#:cidZIFil2;O?.d7Amp,;\L?%9EA.k6GU@5^CJn\_=&TU+%HG-k2"2mK:>_Afe!553&7?k&6QgH6Y(\=>"X3jLXh%0li0RMjIQ&Wipt`aD&d`39%=O$<+>Ae*#8hR95ZRMo&>KqoYR1u$"`KA[l)u.aA*n2uLG/]>ofdnbU"$XTW6lH3l"L9b,O'g4*"(XcXY*'umu<'+22l+g2-FC1W%.>.l38C6at&:TL?*bc(dK!=^E2@'X=3)]neNio)R%^b+4u@s+sA(.Q-=tJ%OFJsbbs`Q,fBBEPKW"Wp(VC\T#U$S#`umZh#40*0=&YN:Z7kNb:\"!"\8D&KNU?s,F=/Qo%='5U-bM5E.Y]&(,0H3qg'M^7BCqfPtoW24S^LsTpc4[$Q3#e`I`BR,HJ3YjNPjM639U7//u8CU"L6LgoHifWRKJ,ltLPh"&ocdT%m1Q0\K'u:cqU\IIYpiF1?1cNGN;N2rtD.RiKkQ>fhro.9A"$31s+m+KA/H9_nlYPs@WonGRq4YWClpETq)WI3ij0;,_>G.,jBsYCL=?Au@AP&1%6BO0Srj.[`dMIGW\NkhB7VKHDMaT]XaN2"a?C9NTCA!hJ`goWBW0mh8k*qJ8_?+K)rB9tSYK`\oR)9s$;)%^B)12I(mB;D"i=6M,?>TFiE:M4<4aA9j2"4[hA@9CPiCdD,(K6ol`/RD[Vk./*1]/7/(5AjeL$#c?g1_$c"@MVuG1\r7cM'`fFA^\0u^;lM)\YYMeO65>Iqt5$o,>kSRsuEj[NXK@*#^+-Jh,9an@=Q+NcOnkcD-!9[QF^TCDsM)KI!6A6ZIlJA4$)qh=H\O"b8,E.uBc*j,U0]<_b4ecJ6\u&?\"6Jui?G(s#PL@aB%Z5C,^qYm,8*@H_WW2l0]Pkh>Pdd\QgcPXt6^tU@6>k8kTaIcL&r$6t/3gUPCmQ5UffG+/IXAGWIi2[!%GF]"e/KLa0$Xd'JDJu!AQ''W(FO@uof=BOMV#tt?N,&dQKDjFSMQae*H#[p.qeGTenI7mo)2_ZX/Bb@FgZq\c=nap]9g9$9R;>Y*nLlVktP?->W]m+=^9--o_A.JAK;J9nSN:WCdZlAg[TF-[eP$F4ht9L]@7n_/O&Xl)TZKh1qKT%?4)c+0m_V,YD-@HCO(Xs56dkK5:0&g+8=sQ3nkG^0^4B&rn:$oRqq@QY3`soF2'l^.[t-PlIcuO\.egX>Vi0.u()84r%BIU]rND]\endstream +endobj +20 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 437 +>> +stream +Gat=e95iQ=&;9NK'lth^"OMlr;3[;L*_d1g-kM0IjW`0CLohs`U^T3^5*-`u4ng_I4ttkFQD'teMu"L)G.>%lY_IK.)_uAnq!PH^lm[;-:4%q_8a4*,<7XrEPQnl%B?\HaWC'[Bj-O)a0@$ZQO=;bd%q`+'jPiq0SuH7g.pIUAd:fs2>?bBuLi6DOMX6tI+M05VP4mpt_Ib1keH:8Z;t*+dE5.\H(981Q*qE$pA;bVEG987Zo^)mD^Q=U3EGl0aR$0cendstream +endobj +xref +0 21 +0000000000 65535 f +0000000061 00000 n +0000000122 00000 n +0000000229 00000 n +0000000341 00000 n +0000000424 00000 n +0000005774 00000 n +0000009789 00000 n +0000009866 00000 n +0000010134 00000 n +0000010339 00000 n +0000010426 00000 n +0000010747 00000 n +0000010821 00000 n +0000010933 00000 n +0000011021 00000 n +0000011128 00000 n +0000011241 00000 n +0000011340 00000 n +0000011406 00000 n +0000014088 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 11 0 R +/Root 10 0 R +/Size 21 +>> +startxref +14616 +%%EOF diff --git a/templates/report.html b/templates/report.html index 2f4b63370b46041e26fba60e180ebd30fc6216ee..287abf9505fc6a64f727d721b2fd3249af3c68af 100644 --- a/templates/report.html +++ b/templates/report.html @@ -15,7 +15,7 @@ /* ── Header / logo row ── */ .header-table { width: 100%; border-collapse: collapse; border-bottom: 2pt solid #4F46E5; padding-bottom: 6pt; margin-bottom: 10pt; } - .logo-cell { font-size: 22pt; font-weight: bold; color: #4F46E5; width: 1%; white-space: nowrap; padding-right: 8pt; } + .logo-cell { font-size: 22pt; font-weight: bold; color: #4F46E5; width: 120pt; white-space: nowrap; padding-right: 8pt; } .logo-shield { color: #6366F1; } .meta-cell { font-size: 8.5pt; color: #6B7280; vertical-align: bottom; } @@ -46,8 +46,8 @@ table.data tr:last-child td { border-bottom: none; } /* ── VLM breakdown ── */ - .vlm-score-bar-wrap { background: #E5E7EB; border-radius: 3pt; height: 5pt; width: 70pt; display: inline-block; vertical-align: middle; overflow: hidden; } - .vlm-score-bar { height: 5pt; border-radius: 3pt; } + .vlm-score-bar-wrap { background: #E5E7EB; height: 5pt; width: 70pt; display: block; overflow: hidden; } + .vlm-score-bar { height: 5pt; display: block; } .vlm-real { background: #43A047; } .vlm-warn { background: #FB8C00; } .vlm-fake { background: #E53935; } diff --git a/test_image_classify.py b/test_image_classify.py deleted file mode 100644 index d38a0b667ed32057de0a08738f461535328648f5..0000000000000000000000000000000000000000 --- a/test_image_classify.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Phase 1.2 smoke test: download a sample image and run the ViT classifier. - -Run from backend/: - .venv/Scripts/python.exe scripts/test_image_classify.py -""" -from __future__ import annotations - -import sys -import urllib.request -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -import base64 - -from models.heatmap_generator import generate_heatmap_base64 -from services.artifact_detector import scan_artifacts -from services.image_service import preprocess_and_classify -from utils.scoring import compute_authenticity_score, get_verdict_label - -SAMPLE_URL = "https://picsum.photos/seed/deepshield/512/512" - - -def main() -> int: - print(f"Fetching sample image: {SAMPLE_URL}") - req = urllib.request.Request(SAMPLE_URL, headers={"User-Agent": "DeepShield/0.1"}) - with urllib.request.urlopen(req, timeout=30) as r: - data = r.read() - print(f" got {len(data)} bytes") - - print("Running classifier (first run will download model ~350MB)…") - pil, result = preprocess_and_classify(data) - print(f" image size: {pil.size}") - print(f" label: {result.label}") - print(f" confidence: {result.confidence:.4f}") - print(f" all scores: {result.all_scores}") - - score = compute_authenticity_score(result.confidence, result.label) - verdict_label, severity = get_verdict_label(score) - print(f"\n authenticity_score: {score}") - print(f" verdict: {verdict_label} ({severity})") - - print("\nScanning artifact indicators\u2026") - for ind in scan_artifacts(pil, data): - print(f" [{ind.severity.upper():6s}] {ind.type}: {ind.description} (conf {ind.confidence:.2f})") - - print("\nGenerating Grad-CAM heatmap\u2026") - heatmap_url = generate_heatmap_base64(pil) - header, b64 = heatmap_url.split(",", 1) - out_path = Path(__file__).resolve().parent.parent / "heatmap_smoketest.png" - out_path.write_bytes(base64.b64decode(b64)) - print(f" saved: {out_path}") - print(f" data URL length: {len(heatmap_url)} chars") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test_news_api.py b/test_news_api.py deleted file mode 100644 index 7efa689a0e80a1d4e4bae7b5e3cfdb85dd7fb030..0000000000000000000000000000000000000000 --- a/test_news_api.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Test script for the NewsData API integration.""" -import asyncio -import sys -import os - -# Add backend directory to sys.path so we can import modules -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) - -from config import settings -from services.news_lookup import search_news_full - -async def test_news(): - print(f"Testing News API Integration with key: {settings.NEWS_API_KEY[:6]}... (masked)") - - if not settings.NEWS_API_KEY: - print("ERROR: NEWS_API_KEY is empty in .env") - return - - keywords = ["modi", "election", "bjp", "congress"] - print(f"Searching for keywords: {keywords}") - - try: - result = await search_news_full(keywords, limit=5) - - print("\n=== RAW RESULT ===") - print(f"Total articles found: {result.total_articles}") - - print("\n=== TRUSTED SOURCES ===") - for i, source in enumerate(result.trusted_sources, 1): - date_str = str(source.published_at)[:10] if source.published_at else "Unknown date" - print(f"{i}. [{source.relevance_score}] {source.source_name}: {source.title[:60]}... ({date_str})") - - print("\n=== CONTRADICTING EVIDENCE / FACT CHECKS ===") - if not result.contradicting_evidence: - print("No fact-check articles found for these keywords.") - for i, ev in enumerate(result.contradicting_evidence, 1): - print(f"{i}. {ev.source_name}: {ev.title[:60]}...") - - except Exception as e: - print(f"\nERROR running test: {e}") - -if __name__ == "__main__": - asyncio.run(test_news()) diff --git a/test_phase5.py b/test_phase5.py deleted file mode 100644 index 5feebb08a02a305c00c093b43e15e083073cf67e..0000000000000000000000000000000000000000 --- a/test_phase5.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Phase 5 smoke: unit-test news_lookup classification + endpoint wiring.""" -from __future__ import annotations - -import asyncio -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from services.news_lookup import ( - _domain_of, _is_factcheck, _relevance, search_news_full, -) - - -def test_domain(): - assert _domain_of("https://www.reuters.com/article/x") == "reuters.com" - assert _domain_of("https://snopes.com/fact-check/abc") == "snopes.com" - print("[OK] _domain_of") - - -def test_factcheck_detection(): - assert _is_factcheck("https://snopes.com/x", "Claim about moon") - assert _is_factcheck("https://factly.in/x", "") - assert _is_factcheck("https://example.com/x", "FACT CHECK: viral video debunked") - assert not _is_factcheck("https://bbc.com/news/world-123", "Election results") - print("[OK] _is_factcheck") - - -def test_relevance(): - assert _relevance("https://reuters.com/x") == 1.0 - assert _relevance("https://ndtv.com/x") == 0.85 - assert _relevance("https://random-blog.xyz/x") == 0.5 - print("[OK] _relevance weights") - - -async def test_empty_key_returns_empty(): - res = await search_news_full(["modi", "election"]) - assert res.trusted_sources == [] - assert res.contradicting_evidence == [] - assert res.total_articles == 0 - print(f"[OK] empty-key path -> {res}") - - -async def test_endpoint_wiring(): - import httpx - body = {"text": "BREAKING!!! You won't BELIEVE this SHOCKING miracle cure doctors don't want you to know!!! Click now!"} - async with httpx.AsyncClient(timeout=180.0) as c: - r = await c.post("http://127.0.0.1:8000/api/v1/analyze/text", json=body) - r.raise_for_status() - j = r.json() - assert j["media_type"] == "text" - assert "trusted_sources" in j - assert "contradicting_evidence" in j - assert "news_lookup" in j["processing_summary"]["stages_completed"] - print(f"[OK] /analyze/text -> verdict={j['verdict']['label']} " - f"score={j['verdict']['authenticity_score']} " - f"trusted={len(j['trusted_sources'])} contradictions={len(j['contradicting_evidence'])}") - - -async def main(): - test_domain() - test_factcheck_detection() - test_relevance() - await test_empty_key_returns_empty() - await test_endpoint_wiring() - print("\n=== Phase 5 smoke PASS ===") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/test_text_analysis.py b/test_text_analysis.py deleted file mode 100644 index 3bedd8f87f8f7be25039e23d5abff27031c5a22c..0000000000000000000000000000000000000000 --- a/test_text_analysis.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Quick smoke test for sensationalism + manipulation detection.""" -import sys -sys.path.insert(0, ".") - -from services.text_service import score_sensationalism, detect_manipulation_indicators - -# --- Sensationalism --- -text1 = "BREAKING: You wont believe this SHOCKING truth! Experts confirm the most DEVASTATING scandal exposed!!!" -s = score_sensationalism(text1) -print(f"Sensationalism: score={s.score} level={s.level}") -print(f" excl={s.exclamation_count} caps={s.caps_word_count} clickbait={s.clickbait_matches} emotional={s.emotional_word_count} superlative={s.superlative_count}") -assert s.score > 50, f"Expected high sensationalism, got {s.score}" -assert s.level in ("Medium", "High"), f"Expected Medium/High, got {s.level}" -print(" PASS") - -# --- Manipulation --- -text2 = "Sources say that experts confirm the shocking truth. Allegedly, everyone knows this is a proven fact." -m = detect_manipulation_indicators(text2) -print(f"\nManipulation indicators: {len(m)} found") -for ind in m: - print(f" [{ind.severity}] {ind.pattern_type}: \"{ind.matched_text}\"") -assert len(m) >= 3, f"Expected >=3 indicators, got {len(m)}" -print(" PASS") - -# --- Clean text --- -text3 = "The weather today is sunny with clear skies in New Delhi." -s2 = score_sensationalism(text3) -m2 = detect_manipulation_indicators(text3) -print(f"\nClean text: sensationalism={s2.score} ({s2.level}), manipulation={len(m2)}") -assert s2.score < 20, f"Expected low sensationalism for clean text, got {s2.score}" -assert len(m2) == 0, f"Expected 0 manipulation indicators for clean text, got {len(m2)}" -print(" PASS") - -print("\nAll tests passed!") diff --git a/tests/test_accuracy_regressions.py b/tests/test_accuracy_regressions.py new file mode 100644 index 0000000000000000000000000000000000000000..4b117ab831527352e64c2192baca1f80fd07054a --- /dev/null +++ b/tests/test_accuracy_regressions.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import os + +os.environ["DEBUG"] = "false" + +from schemas.common import TrustedSource +from services.news_lookup import _compute_truth_override +from services.screenshot_service import OCRBox, extract_full_text +from services.text_service import _scores_to_classification +from utils.scoring import compute_video_authenticity_score +from schemas.common import ArtifactIndicator, ExifSummary, VLMComponentScore, VLMBreakdown +from services.general_image_service import GeneralImageDetection, fuse_no_face_evidence + + +def test_video_score_uses_temporal_and_audio_when_face_content_is_insufficient(): + score, label, severity = compute_video_authenticity_score( + mean_suspicious_prob=0.0, + insufficient_faces=True, + temporal_score=20.0, + audio_authenticity_score=10.0, + has_audio=True, + ) + + assert score < 35 + assert label != "Insufficient face content" + assert severity in {"critical", "danger"} + + +def test_text_classifier_treats_unknown_label_mapping_as_uncertain(): + clf = _scores_to_classification( + [ + {"label": "LABEL_0", "score": 0.99}, + {"label": "LABEL_1", "score": 0.01}, + ], + allow_label0_fallback=False, + ) + + assert clf.fake_prob == 0.5 + assert clf.label == "uncertain_label_mapping" + + +def test_ocr_text_extraction_filters_low_confidence_noise(): + boxes = [ + OCRBox(text="BREAKING", bbox=[[0, 0], [1, 0], [1, 1], [0, 1]], confidence=0.92), + OCRBox(text="x7q", bbox=[[0, 2], [1, 2], [1, 3], [0, 3]], confidence=0.08), + OCRBox(text="confirmed report", bbox=[[0, 4], [1, 4], [1, 5], [0, 5]], confidence=0.51), + ] + + assert extract_full_text(boxes) == "BREAKING confirmed report" + + +def test_truth_override_does_not_apply_from_headline_only_match(monkeypatch): + class FakeSentenceTransformer: + def encode(self, texts, convert_to_numpy=True, normalize_embeddings=True): + import numpy as np + + return np.array([[1.0, 0.0], [1.0, 0.0]], dtype=float) + + class FakeLoader: + def load_sentence_transformer(self): + return FakeSentenceTransformer() + + monkeypatch.setattr("models.model_loader.get_model_loader", lambda: FakeLoader()) + + override = _compute_truth_override( + "Prime Minister announces vaccine drive across Delhi hospitals", + [ + TrustedSource( + source_name="Reuters", + title="Prime Minister announces vaccine drive", + url="https://www.reuters.com/world/example", + relevance_score=1.0, + ) + ], + current_fake_prob=0.9, + ) + + assert override is None or not override.applied + + +def test_no_face_fusion_uses_general_detector_forensic_and_exif_evidence(): + fused = fuse_no_face_evidence( + general_fake_prob=0.72, + artifacts=[ + ArtifactIndicator( + type="gan_artifact", + severity="high", + description="elevated frequency artifacts", + confidence=0.80, + ), + ArtifactIndicator( + type="compression", + severity="medium", + description="unusual compression", + confidence=0.55, + ), + ], + exif=ExifSummary(software="Stable Diffusion", trust_adjustment=10), + ) + + assert fused.fake_probability > 0.72 + assert fused.method == "no_face_general_forensic_fusion" + assert fused.components["general_detector"] == 0.72 + assert fused.components["forensics"] > 0.5 + assert fused.components["exif"] > 0.5 + + +def test_no_face_fusion_can_use_vlm_consistency_scores(): + fused = fuse_no_face_evidence( + general_fake_prob=0.40, + artifacts=[], + exif=None, + vlm=VLMBreakdown( + facial_symmetry=VLMComponentScore(score=80), + skin_texture=VLMComponentScore(score=80), + lighting_consistency=VLMComponentScore(score=25), + background_coherence=VLMComponentScore(score=20), + anatomy_hands_eyes=VLMComponentScore(score=35), + context_objects=VLMComponentScore(score=30), + ), + ) + + assert fused.fake_probability > 0.40 + assert fused.components["vlm_consistency"] > 0.5 + + +def test_no_face_image_route_skips_face_trained_classifiers(monkeypatch): + from PIL import Image + import services.image_service as image_service + + monkeypatch.setattr(image_service, "_has_face_for_routing", lambda _img: False) + monkeypatch.setattr( + image_service, + "classify_general_image", + lambda _img: GeneralImageDetection( + fake_probability=0.8, + label="generated", + all_scores={"generated": 0.8, "real": 0.2}, + model_used="test-general-detector", + ), + ) + monkeypatch.setattr( + image_service, + "_classify_vit", + lambda _img: (_ for _ in ()).throw(AssertionError("face-centric ViT should not run")), + ) + monkeypatch.setattr( + image_service, + "_classify_ffpp", + lambda _img: (_ for _ in ()).throw(AssertionError("FFPP should not run")), + ) + + result = image_service.classify_image(Image.new("RGB", (32, 32), "white")) + + assert result.ensemble_method == "no_face_general_forensic_fusion" + assert result.models_used == ["test-general-detector"] + assert result.no_face_analysis is not None diff --git a/tests/test_efficientnet_regression.py b/tests/test_efficientnet_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..a3d266d3bc4b960405d47d2b933e8539c8db504b --- /dev/null +++ b/tests/test_efficientnet_regression.py @@ -0,0 +1,182 @@ +"""Gate G3 regression harness — EfficientNetAutoAttB4 accuracy on anchor set. + +Acceptance criteria (MERGE_PLAN §9.1 G3): + - >=88% accuracy on the anchor set + - <=8% real->fake false-positive rate + +Anchor set priority: + 1. LOCAL — bundled ICPR2020 notebook/samples/ frames (always available, minimal set) + 2. FFPP — training/datasets/ffpp/ when present (full G3 gate, 50+ images) + 3. DFDC — training/datasets/dfdc/ when present + +NOTE: ThisPersonDoesNotExist.com (StyleGAN2) is NOT valid for G3 — EfficientNetAutoAttB4 +is trained on DFDC video face-swaps and does NOT generalise to GAN-portrait detection. +The full G3 gate requires FFPP c40 data (run scripts/fit_calibrator.py first). + +Run from backend/: + .venv/Scripts/python.exe -m pytest tests/test_efficientnet_regression.py -v +""" +from __future__ import annotations + +import io +import sys +import time +import urllib.request +from pathlib import Path +from typing import Tuple + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +# --------------------------------------------------------------------------- +# Anchor image sources +# --------------------------------------------------------------------------- +# Local: bundled ICPR2020 sample frames (ground-truth labels from their scores). +# lynaeydofd_fr0.jpg → EfficientNet scores 0.011 (REAL) +# mqzvfufzoq_fr0.jpg → EfficientNet scores 0.873 (FAKE) +_ICPR_SAMPLES = ( + Path(__file__).resolve().parent.parent + / "models" / "icpr2020dfdc" / "notebook" / "samples" +) +LOCAL_REAL_IMAGES = [_ICPR_SAMPLES / "lynaeydofd_fr0.jpg"] +LOCAL_FAKE_IMAGES = [_ICPR_SAMPLES / "mqzvfufzoq_fr0.jpg"] + +# FFPP / DFDC local data (full G3 gate — available after running training/datasets download scripts). +_FFPP_REAL = Path(__file__).resolve().parent.parent / "training" / "datasets" / "ffpp" / "c40" / "real" +_FFPP_FAKE = Path(__file__).resolve().parent.parent / "training" / "datasets" / "ffpp" / "c40" / "fake" +_IMAGE_EXTS = {".jpg", ".jpeg", ".png"} + +# Network: thispersondoesnotexist.com — used for G2 gate only (face detection). +# NOT used for G3 accuracy gate: StyleGAN2 faces are a different distribution +# from DFDC video face-swaps (the model's training domain). +TPDNE_URL = "https://thispersondoesnotexist.com/" + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +def _fetch(url: str, timeout: int = 20) -> bytes: + req = urllib.request.Request(url, headers={"User-Agent": "DeepShield-Test/1.0"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read() + + +@pytest.fixture(scope="module") +def detector(): + """Load the EfficientNetDetector once per module.""" + from services.efficientnet_service import EfficientNetDetector + return EfficientNetDetector() + + +@pytest.fixture(scope="module") +def anchor_set(detector) -> Tuple[list, list]: + """Score anchor images. Returns (real_results, fake_results). + + Priority order: + 1. FFPP c40 images (training/datasets/ffpp/c40/{real,fake}/) — full G3 gate + 2. Bundled ICPR2020 notebook samples — minimal sanity check + """ + from PIL import Image + + def score_dir(directory: Path, limit: int = 50) -> list: + results = [] + if not directory.is_dir(): + return results + paths = sorted(p for p in directory.rglob("*") if p.suffix.lower() in _IMAGE_EXTS)[:limit] + for p in paths: + try: + pil = Image.open(p).convert("RGB") + results.append(detector.detect_image(pil)) + except Exception: + pass + return results + + # --- FFPP c40 (full gate) --- + real_results = score_dir(_FFPP_REAL) + fake_results = score_dir(_FFPP_FAKE) + + # --- Fallback: bundled ICPR2020 samples --- + if not real_results: + for p in LOCAL_REAL_IMAGES: + if p.exists(): + pil = Image.open(p).convert("RGB") + real_results.append(detector.detect_image(pil)) + if not fake_results: + for p in LOCAL_FAKE_IMAGES: + if p.exists(): + pil = Image.open(p).convert("RGB") + fake_results.append(detector.detect_image(pil)) + + return real_results, fake_results + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +class TestEfficientNetRegression: + + def test_g1_detector_loads(self, detector): + """G1: model loaded without crash.""" + assert detector is not None + assert detector.net is not None + assert detector.face_extractor is not None + + def test_g2_blazeface_detects_face_on_tpdne(self, detector): + """G2: BlazeFace detects ≥1 face on a TPDNE image.""" + from PIL import Image + data = _fetch(TPDNE_URL) + pil = Image.open(io.BytesIO(data)).convert("RGB") + img_np = np.array(pil) + frame_data = detector.face_extractor.process_image(img=img_np) + faces = frame_data.get("faces", []) + assert len(faces) >= 1, "BlazeFace detected 0 faces on TPDNE image" + + def test_g3_accuracy_and_fpr(self, anchor_set): + """G3: ≥88% accuracy and ≤8% real→fake FPR on the anchor set.""" + real_results, fake_results = anchor_set + + # Only count images where a face was detected (no-face = excluded from G3). + real_scored = [r for r in real_results if r.get("score") is not None] + fake_scored = [r for r in fake_results if r.get("score") is not None] + + if len(real_scored) + len(fake_scored) < 5: + pytest.skip("Too few face-detectable images in anchor set for meaningful G3 check") + + # Correctness: REAL → score ≤ 0.5, FAKE → score > 0.5 + real_correct = sum(1 for r in real_scored if r["result"] == "REAL") + fake_correct = sum(1 for r in fake_scored if r["result"] == "FAKE") + + total = len(real_scored) + len(fake_scored) + accuracy = (real_correct + fake_correct) / total * 100 + + fpr = (len(real_scored) - real_correct) / max(len(real_scored), 1) * 100 + + print(f"\n Anchor set: {len(real_scored)} real | {len(fake_scored)} fake") + print(f" Accuracy: {accuracy:.1f}% (need >=88%)") + print(f" FPR: {fpr:.1f}% (need <=8%)") + for tag, results, exp in [("REAL", real_scored, "REAL"), ("FAKE", fake_scored, "FAKE")]: + for r in results: + mark = "✓" if r["result"] == exp else "✗" + print(f" [{tag}] {mark} score={r['score']:.3f} cal={r.get('calibrator_applied')}") + + assert accuracy >= 88.0, f"G3 accuracy {accuracy:.1f}% < 88%" + assert fpr <= 8.0, f"G3 FPR {fpr:.1f}% > 8%" + + def test_no_face_returns_gracefully(self, detector): + """Noise image with no face should return error='no_face', not raise.""" + from PIL import Image + noise = Image.fromarray(np.random.randint(0, 255, (224, 224, 3), dtype=np.uint8)) + result = detector.detect_image(noise) + assert result["error"] == "no_face" + assert result["score"] is None + + def test_g8_memory_under_threshold(self): + """G8: RSS after model load < 2500 MB.""" + import psutil + rss_mb = psutil.Process().memory_info().rss / 1024 / 1024 + print(f"\n RSS: {rss_mb:.0f} MB") + assert rss_mb < 2500, f"G8: RSS {rss_mb:.0f} MB exceeds 2500 MB threshold" diff --git a/text_service.py b/text_service.py deleted file mode 100644 index 556ac48a8f0195fe5b41d3a789179979e778fae3..0000000000000000000000000000000000000000 --- a/text_service.py +++ /dev/null @@ -1,285 +0,0 @@ -from __future__ import annotations - -import re -from dataclasses import dataclass, field -from typing import List, Optional - -from loguru import logger - -from models.model_loader import get_model_loader - -FAKE_TOKENS = ("fake", "false", "unreliable", "misinformation") - -# --- Sensationalism patterns --- -CLICKBAIT_PATTERNS = [ - (r"\byou won'?t believe\b", "clickbait"), - (r"\bbreaking\s*:", "clickbait"), - (r"\bshocking\s*:", "clickbait"), - (r"\bexclusive\s*:", "clickbait"), - (r"\bjust\s+in\s*:", "clickbait"), - (r"\burgent\s*:", "clickbait"), - (r"\bwhat\s+happens\s+next\b", "clickbait"), - (r"\bthis\s+will\s+change\b", "clickbait"), - (r"\b(?:everyone|nobody)\s+(?:is|was)\s+talking\b", "clickbait"), -] -EMOTIONAL_WORDS = { - "outrage", "shocking", "horrifying", "disgusting", "amazing", "incredible", - "unbelievable", "devastating", "terrifying", "explosive", "bombshell", - "jaw-dropping", "heartbreaking", "furious", "scandal", "crisis", - "chaos", "destroyed", "slammed", "blasted", "exposed", "revealed", -} -SUPERLATIVES = { - "best", "worst", "greatest", "biggest", "most", "least", - "fastest", "deadliest", "largest", "smallest", "ultimate", -} - -# --- Manipulation indicator patterns --- -MANIPULATION_PATTERNS = [ - # Unverified claims - (r"\bsources?\s+(?:say|said|claim|report)\b", "unverified_claim", "medium", - "Unverified source attribution without specific citation"), - (r"\ballegedly\b", "unverified_claim", "low", - "Hedging language suggests unverified information"), - (r"\breports?\s+suggest\b", "unverified_claim", "medium", - "Vague report attribution"), - (r"\baccording\s+to\s+(?:some|many|several)\b", "unverified_claim", "medium", - "Non-specific source attribution"), - (r"\brunconfirmed\b", "unverified_claim", "medium", - "Explicitly unconfirmed information"), - # Emotional manipulation - (r"\boutrage\b", "emotional_manipulation", "medium", - "Emotional trigger word designed to provoke reaction"), - (r"\bshocking\s+truth\b", "emotional_manipulation", "high", - "Sensationalist phrase designed to manipulate reader emotion"), - (r"\bwake\s+up\b", "emotional_manipulation", "medium", - "Call-to-action implying hidden knowledge"), - (r"\bthey\s+don'?t\s+want\s+you\s+to\s+know\b", "emotional_manipulation", "high", - "Conspiracy framing language"), - (r"\bopen\s+your\s+eyes\b", "emotional_manipulation", "medium", - "Implies audience ignorance"), - # False authority - (r"\bexperts?\s+(?:confirm|say|agree|warn)\b", "false_authority", "medium", - "Unnamed expert citation without specific attribution"), - (r"\bscientists?\s+(?:confirm|prove|say)\b", "false_authority", "medium", - "Unnamed scientist citation"), - (r"\bstudies?\s+(?:show|prove|confirm)\b", "false_authority", "low", - "Vague study reference without citation"), - (r"\beveryone\s+knows\b", "false_authority", "medium", - "Appeal to common knowledge fallacy"), - (r"\bit'?s\s+(?:a\s+)?(?:well-?known|proven)\s+fact\b", "false_authority", "medium", - "Assertion of fact without evidence"), -] - -# NER entity labels to prefer for keyword extraction -_NER_PREFERRED = {"PERSON", "ORG", "GPE", "EVENT", "PRODUCT", "NORP"} - - -@dataclass -class TextClassification: - label: str - confidence: float - fake_prob: float - all_scores: dict[str, float] - - -@dataclass -class SensationalismResult: - score: int # 0-100 - level: str # Low / Medium / High - exclamation_count: int - caps_word_count: int - clickbait_matches: int - emotional_word_count: int - superlative_count: int - - -@dataclass -class ManipulationIndicator: - pattern_type: str # unverified_claim / emotional_manipulation / false_authority - matched_text: str - start_pos: int - end_pos: int - severity: str # low / medium / high - description: str - - -def detect_language(text: str) -> str: - """Detect the primary language of text using langdetect. - Returns ISO 639-1 code (e.g. 'en', 'hi'). Falls back to 'en' on failure. - """ - if not text or len(text.strip()) < 10: - return "en" - try: - from langdetect import detect # type: ignore - lang = detect(text.strip()) - logger.info(f"Language detected: {lang}") - return lang - except ImportError: - logger.debug("langdetect not installed — defaulting to 'en'") - return "en" - except Exception as e: - logger.debug(f"Language detection failed: {e} — defaulting to 'en'") - return "en" - - -def _scores_to_classification(items) -> TextClassification: - """Convert pipeline output to TextClassification.""" - scores = {i["label"]: float(i["score"]) for i in items} - top_label, top_conf = max(scores.items(), key=lambda kv: kv[1]) - # Extract fake probability - fake_prob = 0.0 - if "LABEL_0" in scores: - fake_prob = scores["LABEL_0"] - else: - fake_prob = max( - (p for lbl, p in scores.items() if any(t in lbl.lower() for t in FAKE_TOKENS)), - default=0.0, - ) - return TextClassification(top_label, top_conf, fake_prob, scores) - - -def classify_text(text: str, language: Optional[str] = None) -> TextClassification: - """Classify text as fake/real. - Routes to multilingual model when language is non-English and the model is configured. - """ - text = (text or "").strip() - if not text: - return TextClassification("unknown", 0.0, 0.0, {}) - - loader = get_model_loader() - - if language and language != "en": - pipe = loader.load_multilang_text_model() - else: - pipe = loader.load_text_model() - - out = pipe(text[:2000], truncation=True, top_k=None) - items = out[0] if isinstance(out[0], list) else out - clf = _scores_to_classification(items) - logger.info( - f"Text classify [{language or 'en'}] → {clf.label} @ {clf.confidence:.3f} " - f"fake_p={clf.fake_prob:.3f}" - ) - return clf - - -def score_sensationalism(text: str) -> SensationalismResult: - """Compute a 0-100 sensationalism score from structural/linguistic signals.""" - if not text: - return SensationalismResult(0, "Low", 0, 0, 0, 0, 0) - - words = text.split() - total_words = max(len(words), 1) - - excl = text.count("!") - caps = sum(1 for w in words if w.isupper() and len(w) > 2) - clickbait = sum( - 1 for pat, _ in CLICKBAIT_PATTERNS - if re.search(pat, text, re.IGNORECASE) - ) - emotional = sum(1 for w in words if w.lower().strip(".,!?;:") in EMOTIONAL_WORDS) - superlative = sum(1 for w in words if w.lower().strip(".,!?;:") in SUPERLATIVES) - - raw = ( - min(excl * 8, 25) - + min(caps / total_words * 200, 25) - + min(clickbait * 12, 25) - + min(emotional * 6, 15) - + min(superlative * 5, 10) - ) - score = int(min(100, max(0, raw))) - level = "Low" if score < 30 else ("Medium" if score < 60 else "High") - - logger.info(f"Sensationalism → {score} ({level}) excl={excl} caps={caps} cb={clickbait} emo={emotional}") - return SensationalismResult(score, level, excl, caps, clickbait, emotional, superlative) - - -def detect_manipulation_indicators(text: str) -> List[ManipulationIndicator]: - """Scan text for manipulation linguistic patterns with positions.""" - if not text: - return [] - indicators: List[ManipulationIndicator] = [] - for pattern, ptype, severity, description in MANIPULATION_PATTERNS: - for m in re.finditer(pattern, text, re.IGNORECASE): - indicators.append(ManipulationIndicator( - pattern_type=ptype, - matched_text=m.group(), - start_pos=m.start(), - end_pos=m.end(), - severity=severity, - description=description, - )) - indicators.sort(key=lambda i: i.start_pos) - logger.info(f"Manipulation indicators → {len(indicators)} found") - return indicators - - -def extract_entities(text: str, max_k: int = 6) -> List[str]: - """Extract keywords via spaCy NER (PERSON, ORG, GPE, EVENT preferred). - Falls back to frequency-based extraction when spaCy is unavailable or text is too short. - """ - if not text or len(text.strip()) < 20: - return _extract_keywords_freq(text, max_k) - - loader = get_model_loader() - nlp = loader.load_spacy_nlp() - - if nlp is None: - # spaCy not available — use frequency fallback - return _extract_keywords_freq(text, max_k) - - try: - doc = nlp(text[:5000]) # cap for performance - - # Collect named entities, preferring high-value types - preferred: List[str] = [] - other: List[str] = [] - seen: set[str] = set() - - for ent in doc.ents: - norm = ent.text.strip() - norm_lower = norm.lower() - if not norm or norm_lower in seen or len(norm) < 2: - continue - seen.add(norm_lower) - if ent.label_ in _NER_PREFERRED: - preferred.append(norm) - else: - other.append(norm) - - entities = preferred + other - - if len(entities) >= 2: - logger.info(f"NER extracted {len(entities)} entities: {entities[:max_k]}") - return entities[:max_k] - - # Not enough entities — supplement with frequency keywords - freq_kws = _extract_keywords_freq(text, max_k) - combined = entities + [k for k in freq_kws if k.lower() not in seen] - return combined[:max_k] - - except Exception as e: - logger.warning(f"spaCy NER failed: {e} — falling back to frequency extraction") - return _extract_keywords_freq(text, max_k) - - -def _extract_keywords_freq(text: str, max_k: int = 6) -> List[str]: - """Frequency-based keyword extraction (original implementation, kept as fallback).""" - stop = { - "the","a","an","is","are","was","were","be","been","being","to","of","and","or","but", - "in","on","at","for","with","by","from","as","that","this","it","its","has","have","had", - "will","would","can","could","should","may","might","do","does","did","not","no","so", - "than","then","there","their","they","them","we","our","you","your","he","she","his","her", - } - words = re.findall(r"[A-Za-z][A-Za-z\-']{2,}", text or "") - freq: dict[str, int] = {} - for w in words: - wl = w.lower() - if wl in stop: - continue - freq[wl] = freq.get(wl, 0) + 1 - return [w for w, _ in sorted(freq.items(), key=lambda kv: (-kv[1], kv[0]))[:max_k]] - - -# Back-compat alias: routes that still call extract_keywords get NER-first behaviour -extract_keywords = extract_entities diff --git a/__init__.py b/trained_models/.gitkeep similarity index 100% rename from __init__.py rename to trained_models/.gitkeep diff --git a/training/datasets/__init__.py b/training/datasets/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/training/datasets/build_manifest.py b/training/datasets/build_manifest.py deleted file mode 100644 index 00885270483c63cde5aafaffff604f70039084ae..0000000000000000000000000000000000000000 --- a/training/datasets/build_manifest.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Build a unified train/val/test manifest (70/15/15) across all dataset buckets. - -Expected input layout (produced by the other scripts in this package): - - data_root/ - real/ - ffpp_youtube/*.jpg # frames from FFPP original_sequences - ffhq/*.jpg # FFHQ thumbnails - - fake/ - ffpp_deepfakes/*.jpg - ffpp_face2face/*.jpg - ffpp_faceswap/*.jpg - ffpp_neuraltextures/*.jpg - ffpp_faceshifter/*.jpg - dfdc/*.jpg - -The manifest is stratified by (label, source) so FFHQ stays represented -in val/test. - -Usage: - python -m backend.training.datasets.build_manifest \ - --data ./data --out ./data/manifest.csv --seed 42 -""" -from __future__ import annotations - -import argparse -import csv -import random -from collections import defaultdict -from pathlib import Path - -IMG_EXTS = {".jpg", ".jpeg", ".png"} - - -def collect(data_root: Path) -> list[tuple[str, str, str]]: - rows: list[tuple[str, str, str]] = [] - for label in ("real", "fake"): - label_root = data_root / label - if not label_root.exists(): - continue - for source_dir in sorted(p for p in label_root.iterdir() if p.is_dir()): - for img in source_dir.rglob("*"): - if img.suffix.lower() in IMG_EXTS and img.is_file(): - rows.append((str(img.resolve()), label, source_dir.name)) - return rows - - -def split(rows: list[tuple[str, str, str]], seed: int) -> dict[str, list[tuple[str, str, str]]]: - buckets: dict[tuple[str, str], list[tuple[str, str, str]]] = defaultdict(list) - for r in rows: - buckets[(r[1], r[2])].append(r) - - rng = random.Random(seed) - out = {"train": [], "val": [], "test": []} - for key, items in buckets.items(): - rng.shuffle(items) - n = len(items) - n_train = int(0.70 * n) - n_val = int(0.15 * n) - out["train"].extend(items[:n_train]) - out["val"].extend(items[n_train : n_train + n_val]) - out["test"].extend(items[n_train + n_val :]) - return out - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--data", required=True, type=Path) - ap.add_argument("--out", required=True, type=Path) - ap.add_argument("--seed", type=int, default=42) - args = ap.parse_args() - - rows = collect(args.data) - if not rows: - raise SystemExit(f"No images found under {args.data}") - - splits = split(rows, args.seed) - args.out.parent.mkdir(parents=True, exist_ok=True) - with args.out.open("w", newline="", encoding="utf-8") as f: - w = csv.writer(f) - w.writerow(["path", "label", "source", "split"]) - for name, items in splits.items(): - for path, label, source in items: - w.writerow([path, label, source, name]) - - summary = {k: len(v) for k, v in splits.items()} - print(f"Manifest: {args.out}") - print(f"Totals: {summary} (overall {sum(summary.values())})") - - -if __name__ == "__main__": - main() diff --git a/training/datasets/download_dfdc_sample.py b/training/datasets/download_dfdc_sample.py deleted file mode 100644 index 290f639f16430ea03bec20cad71c34b4f4a3a898..0000000000000000000000000000000000000000 --- a/training/datasets/download_dfdc_sample.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Download a sample of the DFDC (Deepfake Detection Challenge) Preview dataset. - -The full DFDC is ~470GB; the *preview* release (~5GB, Kaggle) is enough for -diversity augmentation alongside FFPP. - -Requires the Kaggle CLI (`pip install kaggle`) and ~/.kaggle/kaggle.json. - -Usage: - python -m backend.training.datasets.download_dfdc_sample --output ./data/dfdc_preview -""" -from __future__ import annotations - -import argparse -import shutil -import subprocess -import sys -from pathlib import Path - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--output", required=True, type=Path) - ap.add_argument( - "--competition", - default="deepfake-detection-challenge", - help="Kaggle competition slug (default: deepfake-detection-challenge preview).", - ) - args = ap.parse_args() - - kaggle = shutil.which("kaggle") - if kaggle is None: - print("Kaggle CLI not found. Install with: pip install kaggle", file=sys.stderr) - print("Then place kaggle.json in ~/.kaggle/ (chmod 600).", file=sys.stderr) - sys.exit(2) - - args.output.mkdir(parents=True, exist_ok=True) - cmd = [kaggle, "competitions", "download", "-c", args.competition, "-p", str(args.output)] - print("Running:", " ".join(cmd)) - subprocess.run(cmd, check=True) - print(f"Downloaded to {args.output}. Unzip with: unzip *.zip") - - -if __name__ == "__main__": - main() diff --git a/training/datasets/download_ffhq.py b/training/datasets/download_ffhq.py deleted file mode 100644 index 9aad01da57b77488e4d3113295cd2769c3376826..0000000000000000000000000000000000000000 --- a/training/datasets/download_ffhq.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Download the FFHQ 128x128 thumbnail subset from the official Google Drive mirror. - -Pulls up to N images (default 10k) into the `real` bucket of the training set. -Falls back to the NVlabs 'ffhq-dataset' helper if available; otherwise expects -user to run the manual download once. - -Usage: - python -m backend.training.datasets.download_ffhq --output ./data/real/ffhq -n 10000 -""" -from __future__ import annotations - -import argparse -import shutil -import subprocess -import sys -from pathlib import Path - - -def try_nvlabs_helper(output: Path, num: int) -> bool: - """Prefer the official ffhq-dataset downloader if installed.""" - helper = shutil.which("ffhq-dataset") - if helper is None: - return False - cmd = [helper, "--json", "ffhq-dataset-v2.json", "--thumbs", "--num_threads", "4"] - print("Running:", " ".join(cmd)) - subprocess.run(cmd, cwd=output, check=False) - return True - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--output", required=True, type=Path) - ap.add_argument("-n", "--num", type=int, default=10000) - args = ap.parse_args() - args.output.mkdir(parents=True, exist_ok=True) - - if try_nvlabs_helper(args.output, args.num): - return - - print("[!] `ffhq-dataset` helper not installed.") - print(" Install via: pip install ffhq-dataset (requires gdown)") - print(" Or download thumbnails128x128.zip manually from:") - print(" https://github.com/NVlabs/ffhq-dataset") - print(f" Extract into: {args.output}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/training/datasets/extract_frames.py b/training/datasets/extract_frames.py deleted file mode 100644 index 28bebebb62c2676744a72e1f68ffdf4f5d74c2f4..0000000000000000000000000000000000000000 --- a/training/datasets/extract_frames.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Convert FFPP / DFDC videos -> 16 sampled frames at 224x224 RGB. - -Usage: - python -m backend.training.datasets.extract_frames \ - --input ./ffpp_data/original_sequences/youtube/raw/videos \ - --output ./ffpp_data/frames/real \ - --label real --frames 16 --size 224 -""" -from __future__ import annotations - -import argparse -import csv -from pathlib import Path - -import cv2 -import numpy as np -from tqdm import tqdm - - -def sample_frame_indices(total: int, n: int) -> list[int]: - if total <= 0: - return [] - if total <= n: - return list(range(total)) - step = total / float(n) - return [min(total - 1, int(step * i + step / 2)) for i in range(n)] - - -def extract_from_video(path: Path, out_dir: Path, n: int, size: int) -> int: - cap = cv2.VideoCapture(str(path)) - if not cap.isOpened(): - return 0 - total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - indices = set(sample_frame_indices(total, n)) - out_dir.mkdir(parents=True, exist_ok=True) - - saved = 0 - i = 0 - while True: - ok, frame = cap.read() - if not ok: - break - if i in indices: - frame = cv2.resize(frame, (size, size), interpolation=cv2.INTER_AREA) - cv2.imwrite(str(out_dir / f"{path.stem}_f{i:06d}.jpg"), frame, [cv2.IMWRITE_JPEG_QUALITY, 95]) - saved += 1 - i += 1 - cap.release() - return saved - - -def main() -> None: - ap = argparse.ArgumentParser(description="Sample N frames per video and resize.") - ap.add_argument("--input", required=True, type=Path, help="Directory of .mp4 videos (recursive).") - ap.add_argument("--output", required=True, type=Path, help="Directory to write .jpg frames.") - ap.add_argument("--label", required=True, choices=["real", "fake"], help="Label tag for manifest.") - ap.add_argument("--frames", type=int, default=16) - ap.add_argument("--size", type=int, default=224) - ap.add_argument("--manifest", type=Path, default=None, help="Optional CSV manifest append path.") - args = ap.parse_args() - - videos = [p for p in args.input.rglob("*.mp4")] - if not videos: - print(f"No .mp4 found under {args.input}") - return - - rows: list[tuple[str, str, str]] = [] - total_frames = 0 - for vid in tqdm(videos, desc=f"extract[{args.label}]"): - rel_out = args.output / vid.stem - saved = extract_from_video(vid, rel_out, args.frames, args.size) - total_frames += saved - if args.manifest is not None: - for jpg in rel_out.glob("*.jpg"): - rows.append((str(jpg), args.label, vid.stem)) - - if args.manifest is not None and rows: - args.manifest.parent.mkdir(parents=True, exist_ok=True) - new_file = not args.manifest.exists() - with args.manifest.open("a", newline="", encoding="utf-8") as f: - w = csv.writer(f) - if new_file: - w.writerow(["path", "label", "source_video"]) - w.writerows(rows) - - print(f"Done. Videos: {len(videos)}, frames written: {total_frames}") - - -if __name__ == "__main__": - main() diff --git a/training/datasets/procure_all.ps1 b/training/datasets/procure_all.ps1 deleted file mode 100644 index 4edafaffd1c99062de0244d2c7a3db1cf6813cc6..0000000000000000000000000000000000000000 --- a/training/datasets/procure_all.ps1 +++ /dev/null @@ -1,40 +0,0 @@ -# Phase 11.1 orchestrator for Windows (PowerShell) -$ErrorActionPreference = "Stop" - -$ROOT = if ($env:ROOT) { $env:ROOT } else { ".\data" } -$FFPP = if ($env:FFPP) { $env:FFPP } else { ".\ffpp_data" } - -New-Item -ItemType Directory -Force -Path "$ROOT\real" | Out-Null -New-Item -ItemType Directory -Force -Path "$ROOT\fake" | Out-Null -New-Item -ItemType Directory -Force -Path $FFPP | Out-Null - -Write-Host "1. FaceForensics++ (highly compressed c40, 10 videos only) -- requires TOS keypress" -python backend\scripts\download_ffpp.py $FFPP -d all -c c40 -t videos -n 10 - -Write-Host "2. Frame extraction: real (original youtube)" -python -m backend.training.datasets.extract_frames ` - --input "$FFPP\original_sequences\youtube\c40\videos" ` - --output "$ROOT\real\ffpp_youtube" --label real --frames 4 --size 224 - -Write-Host "3. Frame extraction: fakes (each manipulation family)" -$Families = @("Deepfakes", "Face2Face", "FaceSwap", "NeuralTextures", "FaceShifter") -foreach ($fam in $Families) { - $famLower = $fam.ToLower() - python -m backend.training.datasets.extract_frames ` - --input "$FFPP\manipulated_sequences\$fam\c40\videos" ` - --output "$ROOT\fake\ffpp_$famLower" --label fake --frames 4 --size 224 -} - -Write-Host "4. FFHQ thumbnails (real - limited to 100 items)" -python -m backend.training.datasets.download_ffhq --output "$ROOT\real\ffhq" -n 100 - - -Write-Host "6. DFDC preview sample (fake+real)" -python -m backend.training.datasets.download_dfdc_sample --output "$ROOT\_dfdc_raw" -Write-Host "NOTE: You will need to manually unzip + sort DFDC into $ROOT\fake\dfdc and $ROOT\real\dfdc" - -Write-Host "7. Build manifest" -python -m backend.training.datasets.build_manifest ` - --data $ROOT --out "$ROOT\manifest.csv" --seed 42 - -Write-Host "Phase 11.1 complete. See $ROOT\manifest.csv" diff --git a/training/datasets/procure_all.sh b/training/datasets/procure_all.sh deleted file mode 100644 index fa2f94ecf2959a92df56d31572c4c26c9256b0d1..0000000000000000000000000000000000000000 --- a/training/datasets/procure_all.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# Phase 11.1 orchestrator: download + frame-extract + manifest. -# Total disk target: ~120k labeled images. Expect 60-80GB intermediate, ~30GB frames. - -set -euo pipefail - -ROOT="${ROOT:-./data}" -FFPP="${FFPP:-./ffpp_data}" -mkdir -p "$ROOT/real" "$ROOT/fake" "$FFPP" - -# 1. FaceForensics++ (raw, videos) -- requires TOS keypress -python backend/scripts/download_ffpp.py "$FFPP" -d all -c raw -t videos - -# 2. Frame extraction: real (original youtube) -python -m backend.training.datasets.extract_frames \ - --input "$FFPP/original_sequences/youtube/raw/videos" \ - --output "$ROOT/real/ffpp_youtube" --label real --frames 16 --size 224 - -# 3. Frame extraction: fakes (each manipulation family) -for fam in Deepfakes Face2Face FaceSwap NeuralTextures FaceShifter; do - python -m backend.training.datasets.extract_frames \ - --input "$FFPP/manipulated_sequences/$fam/raw/videos" \ - --output "$ROOT/fake/ffpp_${fam,,}" --label fake --frames 16 --size 224 -done - -# 4. FFHQ thumbnails (real) -python -m backend.training.datasets.download_ffhq --output "$ROOT/real/ffhq" -n 10000 - -# 6. DFDC preview sample (fake+real) -- needs Kaggle creds -python -m backend.training.datasets.download_dfdc_sample --output "$ROOT/_dfdc_raw" -# NOTE: unzip + sort into $ROOT/fake/dfdc and $ROOT/real/dfdc per DFDC metadata.json - -# 7. Build manifest -python -m backend.training.datasets.build_manifest \ - --data "$ROOT" --out "$ROOT/manifest.csv" --seed 42 - -echo "Phase 11.1 complete. See $ROOT/manifest.csv" diff --git a/utils/file_handler.py b/utils/file_handler.py index dc88cbed4ecce474c2eb7b5d543bdd8ae9124717..29f15128dae85f8e3eb709778b7a950d23b198bc 100644 --- a/utils/file_handler.py +++ b/utils/file_handler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import io import os import tempfile @@ -7,8 +8,6 @@ from typing import Iterable from fastapi import HTTPException, UploadFile, status -from config import settings - IMAGE_MAGIC_BYTES: dict[bytes, str] = { b"\xff\xd8\xff": "image/jpeg", b"\x89PNG\r\n\x1a\n": "image/png", @@ -34,6 +33,11 @@ async def read_upload_bytes( Returns (raw_bytes, detected_mime). Raises HTTPException on failure. """ data = await file.read() + if not data: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Empty file — no bytes received", + ) size_mb = len(data) / (1024 * 1024) if size_mb > max_size_mb: raise HTTPException( @@ -86,7 +90,7 @@ async def save_upload_to_tempfile( status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail=f"File too large (> {max_size_mb} MB)", ) - out.write(chunk) + await asyncio.to_thread(out.write, chunk) except Exception: try: os.unlink(path) diff --git a/utils/scoring.py b/utils/scoring.py index eec7009e2d63204fd7952bc3e0d30afb561dacc1..1f873fe13d942064d653ba3d104049e24f541a2a 100644 --- a/utils/scoring.py +++ b/utils/scoring.py @@ -11,17 +11,15 @@ TRUST_SCALE = [ ] -def compute_authenticity_score(model_confidence: float, label: str) -> int: - """Map (confidence, label) to 0-100 authenticity score. - Real-ish labels give high score; fake-ish labels give low score. +def compute_authenticity_score(fake_probability: float, label: str = "") -> int: + """Map a fake probability [0.0, 1.0] to a 0-100 authenticity score. + + The first argument must always be the model's fake-probability (not the + top-label confidence). 0.0 (no fake signal) → 100, 1.0 (certain fake) → 0. + + The `label` parameter is accepted for backward compatibility but not used. """ - label_l = label.lower() - fake_tokens = ("fake", "deepfake", "manipulated", "ai", "generated", "synthetic") - if any(tok in label_l for tok in fake_tokens): - score = (1.0 - float(model_confidence)) * 100.0 - else: - score = float(model_confidence) * 100.0 - return int(round(max(0.0, min(100.0, score)))) + return int(round(max(0.0, min(100.0, (1.0 - float(fake_probability)) * 100.0)))) def get_verdict_label(score: int) -> Tuple[str, str]: @@ -31,6 +29,47 @@ def get_verdict_label(score: int) -> Tuple[str, str]: return "Unknown", "warning" +def compute_video_authenticity_score( + *, + mean_suspicious_prob: float, + insufficient_faces: bool, + temporal_score: float | None = None, + audio_authenticity_score: float | None = None, + has_audio: bool = False, +) -> Tuple[int, str, str]: + """Combine video evidence into an authenticity verdict. + + Face-model evidence is authoritative only when enough face frames were + scored. If face content is insufficient, use temporal/audio evidence when + available instead of forcing a neutral result. + """ + if insufficient_faces: + evidence: list[tuple[float, float]] = [] + if temporal_score is not None: + evidence.append((0.60, float(temporal_score))) + if has_audio and audio_authenticity_score is not None: + evidence.append((0.40, float(audio_authenticity_score))) + + if not evidence: + return 50, "Insufficient face content", "warning" + + total_weight = sum(weight for weight, _score in evidence) + combined = sum(weight * score for weight, score in evidence) / total_weight + score = int(round(max(0.0, min(100.0, combined)))) + label, severity = get_verdict_label(score) + return score, label, severity + + visual_score = (1.0 - float(mean_suspicious_prob)) * 100.0 + temporal_sc = float(temporal_score) if temporal_score is not None else visual_score + if has_audio and audio_authenticity_score is not None: + combined = 0.50 * visual_score + 0.30 * temporal_sc + 0.20 * float(audio_authenticity_score) + else: + combined = 0.70 * visual_score + 0.30 * temporal_sc + score = int(round(max(0.0, min(100.0, combined)))) + label, severity = get_verdict_label(score) + return score, label, severity + + def get_score_color(score: int) -> str: """Linear interpolate Red (#E53935) → Amber (#FFA726) → Green (#43A047).""" def lerp(a: int, b: int, t: float) -> int: diff --git a/v1/__init__.py b/v1/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/v1/analyze.py b/v1/analyze.py deleted file mode 100644 index 1ca2f714f552720a6b0655f0d0d616251387cee0..0000000000000000000000000000000000000000 --- a/v1/analyze.py +++ /dev/null @@ -1,605 +0,0 @@ -from __future__ import annotations - -import json -import os -import time -import uuid -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends, File, UploadFile -from pydantic import BaseModel -from loguru import logger -from sqlalchemy.orm import Session - -from api.deps import optional_current_user -from config import settings -from db.database import get_db -from db.models import AnalysisRecord, User -from models.heatmap_generator import generate_heatmap_base64, generate_boxes_base64 -from schemas.analyze import ( - FrameAnalysisOut, - ImageAnalysisResponse, - ImageExplainability, - LayoutAnomalyOut, - ManipulationIndicatorOut, - OCRBoxOut, - ScreenshotAnalysisResponse, - ScreenshotExplainability, - SensationalismBreakdown, - SuspiciousPhraseOut, - TextAnalysisResponse, - TextExplainability, - VideoAnalysisResponse, - VideoExplainability, -) -from services.screenshot_service import ( - detect_layout_anomalies, - extract_full_text, - map_phrases_to_boxes, - run_ocr, -) -from services.ela_service import generate_ela_base64 -from services.exif_service import extract_exif -from services.image_service import load_image_from_bytes -from services.llm_explainer import generate_llm_summary -from schemas.common import ProcessingSummary, Verdict -from services.artifact_detector import scan_artifacts -from services.image_service import preprocess_and_classify -from services.news_lookup import search_news_full -from services.vlm_breakdown import generate_vlm_breakdown -from services.text_service import ( - classify_text, - detect_language, - detect_manipulation_indicators, - extract_entities, - score_sensationalism, -) -from services.video_service import analyze_video -from services.metadata_writer import write_verdict_metadata -from utils.file_handler import read_upload_bytes, save_upload_to_tempfile -from utils.scoring import compute_authenticity_score, get_verdict_label - -router = APIRouter(prefix="/analyze", tags=["analyze"]) - -IMAGE_MAX_MB = 20 -VIDEO_MAX_MB = 100 -VIDEO_NUM_FRAMES = 16 - - -@router.post("/image", response_model=ImageAnalysisResponse) -async def analyze_image( - file: UploadFile = File(...), - db: Session = Depends(get_db), - user: User | None = Depends(optional_current_user), -) -> ImageAnalysisResponse: - start = time.perf_counter() - stages: list[str] = [] - - raw, mime = await read_upload_bytes( - file, settings.ALLOWED_IMAGE_TYPES, max_size_mb=IMAGE_MAX_MB - ) - stages.append("validation") - - pil, clf = preprocess_and_classify(raw) - stages.append("classification") - - indicators = scan_artifacts(pil, raw) - stages.append("artifact_scanning") - - # ── Phase 12: Grad-CAM++ heatmap ── - heatmap_status = "success" - heatmap = "" - try: - model_family = "efficientnet" if settings.ENSEMBLE_MODE else "vit" - heatmap, heatmap_source = generate_heatmap_base64(pil, model_family=model_family) - if not heatmap: - heatmap_status = heatmap_source # "none" or "fallback" - stages.append("heatmap_generation") - except Exception as e: # noqa: BLE001 - logger.warning(f"Heatmap generation failed, continuing: {e}") - heatmap_status = "failed" - - # ── Phase 12: ELA (Error Level Analysis) ── - ela_b64 = "" - try: - ela_b64 = generate_ela_base64(pil) - stages.append("ela_generation") - except Exception as e: # noqa: BLE001 - logger.warning(f"ELA generation failed, continuing: {e}") - - # ── Phase 12: Bounding box mode ── - boxes_b64 = "" - try: - boxes_b64 = generate_boxes_base64(pil) - stages.append("boxes_generation") - except Exception as e: # noqa: BLE001 - logger.warning(f"Bounding box generation failed, continuing: {e}") - - # ── Phase 12: EXIF extraction + trust adjustment ── - exif_summary = None - try: - exif_summary = extract_exif(pil, raw) - stages.append("exif_extraction") - except Exception as e: # noqa: BLE001 - logger.warning(f"EXIF extraction failed, continuing: {e}") - - score = compute_authenticity_score(clf.confidence, clf.label) - - # Apply EXIF trust adjustment to the score - if exif_summary and exif_summary.trust_adjustment != 0: - score = int(round(max(0, min(100, score + exif_summary.trust_adjustment)))) - - label, severity = get_verdict_label(score) - duration_ms = int((time.perf_counter() - start) * 1000) - - analysis_id = str(uuid.uuid4()) - - response = ImageAnalysisResponse( - analysis_id=analysis_id, - media_type="image", - timestamp=datetime.now(timezone.utc).isoformat(), - verdict=Verdict( - label=label, - severity=severity, - authenticity_score=score, - model_confidence=clf.confidence, - model_label=clf.label, - ), - explainability=ImageExplainability( - heatmap_base64=heatmap, - ela_base64=ela_b64, - boxes_base64=boxes_b64, - heatmap_status=heatmap_status, - artifact_indicators=indicators, - exif=exif_summary, - ), - trusted_sources=[], - contradicting_evidence=[], - processing_summary=ProcessingSummary( - stages_completed=stages, - total_duration_ms=duration_ms, - model_used=settings.IMAGE_MODEL_ID, - models_used=clf.models_used, - ), - ) - - record = AnalysisRecord( - user_id=user.id if user else None, - media_type="image", - verdict=label, - authenticity_score=float(score), - result_json=json.dumps(response.model_dump( - exclude={"explainability": {"heatmap_base64", "ela_base64", "boxes_base64"}} - )), - ) - db.add(record) - db.commit() - db.refresh(record) - response.record_id = record.id - logger.info(f"Saved AnalysisRecord id={record.id} score={score} verdict={label}") - - # ── Phase 12: LLM explainability card (runs after DB save so we have record_id) ── - try: - llm_summary = generate_llm_summary( - payload=response.model_dump( - exclude={"explainability": {"heatmap_base64", "ela_base64", "boxes_base64"}} - ), - record_id=str(record.id), - ) - response.explainability.llm_summary = llm_summary - stages.append("llm_explanation") - except Exception as e: # noqa: BLE001 - logger.warning(f"LLM explainer failed, continuing: {e}") - - # ── Phase 14: VLM detailed breakdown (vision LLM scores 6 perceptual components) ── - try: - vlm_bd = generate_vlm_breakdown(pil, record_id=str(record.id)) - if vlm_bd: - response.explainability.vlm_breakdown = vlm_bd - stages.append("vlm_breakdown") - except Exception as e: # noqa: BLE001 - logger.warning(f"VLM breakdown failed, continuing: {e}") - - return response - - -@router.post("/video", response_model=VideoAnalysisResponse) -async def analyze_video_endpoint( - file: UploadFile = File(...), - db: Session = Depends(get_db), - user: User | None = Depends(optional_current_user), -) -> VideoAnalysisResponse: - start = time.perf_counter() - stages: list[str] = [] - - suffix = os.path.splitext(file.filename or "")[1].lower() or ".mp4" - path, mime = await save_upload_to_tempfile( - file, settings.ALLOWED_VIDEO_TYPES, max_size_mb=VIDEO_MAX_MB, suffix=suffix - ) - stages.append("validation") - - try: - agg = analyze_video(path, num_frames=VIDEO_NUM_FRAMES) - stages.append("frame_extraction") - stages.append("frame_classification") - stages.append("aggregation") - except Exception: - try: - os.unlink(path) - except OSError: - pass - raise - - if agg.insufficient_faces: - score = 50 - label = "Insufficient face content" - severity = "warning" - else: - score = int(round(max(0.0, min(100.0, (1.0 - agg.mean_suspicious_prob) * 100.0)))) - label, severity = get_verdict_label(score) - duration_ms = int((time.perf_counter() - start) * 1000) - - response = VideoAnalysisResponse( - analysis_id=str(uuid.uuid4()), - media_type="video", - timestamp=datetime.now(timezone.utc).isoformat(), - verdict=Verdict( - label=label, - severity=severity, - authenticity_score=score, - model_confidence=float(agg.mean_suspicious_prob), - model_label="suspicious_mean" if not agg.insufficient_faces else "no_faces", - ), - explainability=VideoExplainability( - num_frames_sampled=agg.num_frames_sampled, - num_face_frames=agg.num_face_frames, - num_suspicious_frames=agg.num_suspicious_frames, - mean_suspicious_prob=agg.mean_suspicious_prob, - max_suspicious_prob=agg.max_suspicious_prob, - suspicious_ratio=agg.suspicious_ratio, - insufficient_faces=agg.insufficient_faces, - suspicious_timestamps=agg.suspicious_timestamps, - frames=[ - FrameAnalysisOut( - index=f.index, - timestamp_s=f.timestamp_s, - label=f.label, - confidence=f.confidence, - suspicious_prob=f.suspicious_prob, - is_suspicious=f.is_suspicious, - has_face=f.has_face, - scored=f.scored, - ) - for f in agg.frames - ], - ), - processing_summary=ProcessingSummary( - stages_completed=stages, - total_duration_ms=duration_ms, - model_used=settings.IMAGE_MODEL_ID, - models_used=agg.models_used, - ), - ) - - record = AnalysisRecord( - user_id=user.id if user else None, - media_type="video", - verdict=label, - authenticity_score=float(score), - result_json=json.dumps(response.model_dump()), - ) - db.add(record) - db.commit() - db.refresh(record) - response.record_id = record.id - logger.info( - f"Saved AnalysisRecord id={record.id} video score={score} verdict={label} " - f"frames={agg.num_frames_sampled} susp={agg.num_suspicious_frames}" - ) - - # Write verdict into video metadata (ExifTool, optional — gated by EXIFTOOL_PATH). - try: - write_verdict_metadata( - file_path=path, - verdict=label, - authenticity_score=score, - models_used=agg.models_used, - analysis_id=str(record.id), - ) - except Exception as e: # noqa: BLE001 - logger.warning(f"Metadata write failed: {e}") - finally: - try: - os.unlink(path) - except OSError: - pass - - # Phase 12: LLM explainability card - try: - response.llm_summary = generate_llm_summary( - payload=response.model_dump(), record_id=str(record.id), - ) - except Exception as e: # noqa: BLE001 - logger.warning(f"LLM explainer failed for video: {e}") - - return response - - -class TextAnalyzeBody(BaseModel): - text: str - - -@router.post("/text", response_model=TextAnalysisResponse) -async def analyze_text_endpoint( - body: TextAnalyzeBody = Body(...), - db: Session = Depends(get_db), - user: User | None = Depends(optional_current_user), -) -> TextAnalysisResponse: - start = time.perf_counter() - stages: list[str] = [] - - # Phase 13: language detection — routes to multilang model when non-English - lang = detect_language(body.text) - stages.append("language_detection") - - clf = classify_text(body.text, language=lang) - stages.append("classification") - - sens = score_sensationalism(body.text) - stages.append("sensationalism_analysis") - - manip = detect_manipulation_indicators(body.text) - stages.append("manipulation_detection") - - # Phase 13.1: NER-based keyword extraction (spaCy entities first, frequency fallback) - keywords = extract_entities(body.text) - stages.append("ner_keyword_extraction") - - # Phase 13.2: pass original text + current fake_prob for truth-override computation - news = await search_news_full( - keywords, - original_text=body.text, - current_fake_prob=clf.fake_prob, - ) - stages.append("news_lookup") - - # Apply truth-override to fake_prob before scoring - effective_fake_prob = clf.fake_prob - if news.truth_override and news.truth_override.applied: - effective_fake_prob = news.truth_override.fake_prob_after - stages.append("truth_override_applied") - - # Weighted score: 70% classifier + 20% inverse sensationalism + 10% manipulation penalty - manip_penalty = min(len(manip) * 5, 30) - raw_score = (1.0 - effective_fake_prob) * 100.0 - weighted = raw_score * 0.70 + max(0, 100 - sens.score) * 0.20 + max(0, 100 - manip_penalty) * 0.10 - score = int(round(max(0.0, min(100.0, weighted)))) - label, severity = get_verdict_label(score) - duration_ms = int((time.perf_counter() - start) * 1000) - - model_used = ( - settings.TEXT_MULTILANG_MODEL_ID if (lang != "en" and settings.TEXT_MULTILANG_MODEL_ID) - else settings.TEXT_MODEL_ID - ) - - response = TextAnalysisResponse( - analysis_id=str(uuid.uuid4()), - media_type="text", - timestamp=datetime.now(timezone.utc).isoformat(), - verdict=Verdict( - label=label, - severity=severity, - authenticity_score=score, - model_confidence=float(clf.confidence), - model_label=clf.label, - ), - explainability=TextExplainability( - fake_probability=effective_fake_prob, - top_label=clf.label, - all_scores=clf.all_scores, - keywords=keywords, - sensationalism=SensationalismBreakdown( - score=sens.score, - level=sens.level, - exclamation_count=sens.exclamation_count, - caps_word_count=sens.caps_word_count, - clickbait_matches=sens.clickbait_matches, - emotional_word_count=sens.emotional_word_count, - superlative_count=sens.superlative_count, - ), - manipulation_indicators=[ - ManipulationIndicatorOut( - pattern_type=m.pattern_type, - matched_text=m.matched_text, - start_pos=m.start_pos, - end_pos=m.end_pos, - severity=m.severity, - description=m.description, - ) - for m in manip - ], - detected_language=lang, - truth_override=news.truth_override, - ), - trusted_sources=news.trusted_sources, - contradicting_evidence=news.contradicting_evidence, - processing_summary=ProcessingSummary( - stages_completed=stages, - total_duration_ms=duration_ms, - model_used=model_used, - ), - ) - - record = AnalysisRecord( - user_id=user.id if user else None, - media_type="text", - verdict=label, - authenticity_score=float(score), - result_json=json.dumps(response.model_dump()), - ) - db.add(record) - db.commit() - db.refresh(record) - response.record_id = record.id - logger.info(f"Saved AnalysisRecord id={record.id} text score={score} verdict={label}") - - # Phase 12: LLM explainability card - try: - response.llm_summary = generate_llm_summary( - payload=response.model_dump(), record_id=str(record.id), - ) - except Exception as e: # noqa: BLE001 - logger.warning(f"LLM explainer failed for text: {e}") - - return response - - -@router.post("/screenshot", response_model=ScreenshotAnalysisResponse) -async def analyze_screenshot_endpoint( - file: UploadFile = File(...), - db: Session = Depends(get_db), - user: User | None = Depends(optional_current_user), -) -> ScreenshotAnalysisResponse: - start = time.perf_counter() - stages: list[str] = [] - - raw, mime = await read_upload_bytes( - file, settings.ALLOWED_IMAGE_TYPES, max_size_mb=IMAGE_MAX_MB - ) - stages.append("validation") - - pil = load_image_from_bytes(raw) - ocr_boxes = run_ocr(pil) - stages.append("ocr") - - full_text = extract_full_text(ocr_boxes) - - # Phase 13: language detection on extracted OCR text - lang = detect_language(full_text) if full_text else "en" - stages.append("language_detection") - - clf = classify_text(full_text, language=lang) if full_text else None - stages.append("classification") - - sens = score_sensationalism(full_text) - stages.append("sensationalism_analysis") - - manip = detect_manipulation_indicators(full_text) - stages.append("manipulation_detection") - - phrases = map_phrases_to_boxes(ocr_boxes, manip) - stages.append("phrase_overlay_mapping") - - layout = detect_layout_anomalies(ocr_boxes) - stages.append("layout_anomaly_detection") - - # Phase 13.1: NER-based keyword extraction - keywords = extract_entities(full_text) - stages.append("ner_keyword_extraction") - - fake_prob = clf.fake_prob if clf else 0.0 - model_conf = clf.confidence if clf else 0.0 - model_lbl = clf.label if clf else "no_text" - - # Phase 13.2: truth-override via cosine similarity - news = await search_news_full( - keywords, - original_text=full_text, - current_fake_prob=fake_prob, - ) - stages.append("news_lookup") - - effective_fake_prob = fake_prob - if news.truth_override and news.truth_override.applied: - effective_fake_prob = news.truth_override.fake_prob_after - stages.append("truth_override_applied") - - manip_penalty = min(len(manip) * 5, 30) - layout_penalty = min(len(layout) * 5, 15) - raw_score = (1.0 - effective_fake_prob) * 100.0 - weighted = ( - raw_score * 0.65 - + max(0, 100 - sens.score) * 0.20 - + max(0, 100 - manip_penalty) * 0.10 - + max(0, 100 - layout_penalty) * 0.05 - ) - if not full_text.strip(): - weighted = 50 - score = int(round(max(0.0, min(100.0, weighted)))) - label, severity = get_verdict_label(score) - duration_ms = int((time.perf_counter() - start) * 1000) - - model_used_str = ( - f"{settings.TEXT_MULTILANG_MODEL_ID} + EasyOCR" - if (lang != "en" and settings.TEXT_MULTILANG_MODEL_ID) - else f"{settings.TEXT_MODEL_ID} + EasyOCR" - ) - - response = ScreenshotAnalysisResponse( - analysis_id=str(uuid.uuid4()), - media_type="screenshot", - timestamp=datetime.now(timezone.utc).isoformat(), - verdict=Verdict( - label=label, - severity=severity, - authenticity_score=score, - model_confidence=float(model_conf), - model_label=model_lbl, - ), - explainability=ScreenshotExplainability( - extracted_text=full_text, - ocr_boxes=[OCRBoxOut(text=b.text, bbox=b.bbox, confidence=b.confidence) for b in ocr_boxes], - fake_probability=effective_fake_prob, - sensationalism=SensationalismBreakdown( - score=sens.score, level=sens.level, - exclamation_count=sens.exclamation_count, caps_word_count=sens.caps_word_count, - clickbait_matches=sens.clickbait_matches, emotional_word_count=sens.emotional_word_count, - superlative_count=sens.superlative_count, - ), - suspicious_phrases=[ - SuspiciousPhraseOut( - text=p.text, bbox=p.bbox, pattern_type=p.pattern_type, - severity=p.severity, description=p.description, - ) for p in phrases - ], - layout_anomalies=[ - LayoutAnomalyOut( - type=la.type, severity=la.severity, - description=la.description, confidence=la.confidence, - ) for la in layout - ], - keywords=keywords, - detected_language=lang, - truth_override=news.truth_override, - ), - trusted_sources=news.trusted_sources, - contradicting_evidence=news.contradicting_evidence, - processing_summary=ProcessingSummary( - stages_completed=stages, - total_duration_ms=duration_ms, - model_used=model_used_str, - ), - ) - - record = AnalysisRecord( - user_id=user.id if user else None, - media_type="screenshot", - verdict=label, - authenticity_score=float(score), - result_json=json.dumps(response.model_dump()), - ) - db.add(record) - db.commit() - db.refresh(record) - response.record_id = record.id - logger.info(f"Saved AnalysisRecord id={record.id} screenshot score={score} verdict={label}") - - # Phase 12: LLM explainability card - try: - response.llm_summary = generate_llm_summary( - payload=response.model_dump(), record_id=str(record.id), - ) - except Exception as e: # noqa: BLE001 - logger.warning(f"LLM explainer failed for screenshot: {e}") - - return response diff --git a/v1/auth.py b/v1/auth.py deleted file mode 100644 index c8e61ca0cbc138c8e1e2b7420996e4892e46468e..0000000000000000000000000000000000000000 --- a/v1/auth.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -from fastapi import APIRouter, Depends, HTTPException, status -from loguru import logger -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session - -from api.deps import get_current_user -from config import settings -from db.database import get_db -from db.models import User -from schemas.auth import LoginBody, RegisterBody, TokenResponse, UserOut -from services.auth_service import authenticate, create_access_token, register_user - -router = APIRouter(prefix="/auth", tags=["auth"]) - - -def _token_response(user: User) -> TokenResponse: - return TokenResponse( - access_token=create_access_token(user.id, user.email), - expires_in_minutes=settings.JWT_EXPIRATION_MINUTES, - user=UserOut(id=user.id, email=user.email, name=user.name, created_at=user.created_at), - ) - - -@router.post("/register", response_model=TokenResponse, status_code=status.HTTP_201_CREATED) -def register(body: RegisterBody, db: Session = Depends(get_db)) -> TokenResponse: - try: - user = register_user(db, body.email, body.password, body.name) - except IntegrityError: - db.rollback() - raise HTTPException(status.HTTP_409_CONFLICT, "Email already registered") - logger.info(f"Registered user id={user.id} email={user.email}") - return _token_response(user) - - -@router.post("/login", response_model=TokenResponse) -def login(body: LoginBody, db: Session = Depends(get_db)) -> TokenResponse: - user = authenticate(db, body.email, body.password) - if not user: - raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid email or password") - logger.info(f"Login user id={user.id} email={user.email}") - return _token_response(user) - - -@router.get("/me", response_model=UserOut) -def me(user: User = Depends(get_current_user)) -> UserOut: - return UserOut(id=user.id, email=user.email, name=user.name, created_at=user.created_at) diff --git a/v1/health.py b/v1/health.py deleted file mode 100644 index b02fd9845b16997bda02ffe00e91915c4c043533..0000000000000000000000000000000000000000 --- a/v1/health.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi import APIRouter - -router = APIRouter(tags=["health"]) - - -@router.get("/health") -def health(): - return {"status": "ok", "service": "deepshield-backend"} diff --git a/v1/history.py b/v1/history.py deleted file mode 100644 index db70c77e068a5e4f8070caddc011504868912493..0000000000000000000000000000000000000000 --- a/v1/history.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -import json -from datetime import datetime - -from fastapi import APIRouter, Depends, HTTPException, Query, status -from pydantic import BaseModel -from sqlalchemy.orm import Session - -from api.deps import get_current_user -from db.database import get_db -from db.models import AnalysisRecord, User - -router = APIRouter(prefix="/history", tags=["history"]) - - -class HistoryItem(BaseModel): - id: int - media_type: str - verdict: str - authenticity_score: float - created_at: datetime - - -class HistoryListResponse(BaseModel): - items: list[HistoryItem] - total: int - - -@router.get("", response_model=HistoryListResponse) -def list_history( - limit: int = Query(default=50, ge=1, le=200), - offset: int = Query(default=0, ge=0), - user: User = Depends(get_current_user), - db: Session = Depends(get_db), -) -> HistoryListResponse: - q = db.query(AnalysisRecord).filter(AnalysisRecord.user_id == user.id) - total = q.count() - rows = q.order_by(AnalysisRecord.created_at.desc()).offset(offset).limit(limit).all() - items = [ - HistoryItem( - id=r.id, - media_type=r.media_type, - verdict=r.verdict, - authenticity_score=r.authenticity_score, - created_at=r.created_at, - ) - for r in rows - ] - return HistoryListResponse(items=items, total=total) - - -@router.get("/{record_id}") -def get_history_detail( - record_id: int, - user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - r = db.query(AnalysisRecord).filter(AnalysisRecord.id == record_id).first() - if not r or r.user_id != user.id: - raise HTTPException(status.HTTP_404_NOT_FOUND, "Analysis not found") - try: - return json.loads(r.result_json) - except Exception: - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Corrupt result payload") - - -@router.delete("/{record_id}", status_code=status.HTTP_204_NO_CONTENT) -def delete_history( - record_id: int, - user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - r = db.query(AnalysisRecord).filter(AnalysisRecord.id == record_id).first() - if not r or r.user_id != user.id: - raise HTTPException(status.HTTP_404_NOT_FOUND, "Analysis not found") - db.delete(r) - db.commit() - return None diff --git a/v1/report.py b/v1/report.py deleted file mode 100644 index 72a34c8165dbd78f8e474afdc6d9df77d6e54494..0000000000000000000000000000000000000000 --- a/v1/report.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from fastapi import APIRouter, Depends, HTTPException -from fastapi.responses import FileResponse -from loguru import logger -from sqlalchemy.orm import Session - -from db.database import get_db -from db.models import AnalysisRecord, Report -from services.report_service import cleanup_expired, create_report_row, generate_report - -router = APIRouter(prefix="/report", tags=["report"]) - - -@router.post("/{analysis_id}") -def generate(analysis_id: int, db: Session = Depends(get_db)): - record = db.query(AnalysisRecord).filter(AnalysisRecord.id == analysis_id).first() - if not record: - raise HTTPException(status_code=404, detail="analysis not found") - - existing = db.query(Report).filter(Report.analysis_id == analysis_id).first() - if existing and Path(existing.file_path).exists(): - return {"report_id": existing.id, "analysis_id": analysis_id, "ready": True} - - try: - path = generate_report(record) - except Exception as e: # noqa: BLE001 - logger.exception(f"Report generation failed: {e}") - raise HTTPException(status_code=500, detail=f"report generation failed: {e}") - - if existing: - existing.file_path = str(path) - db.commit() - db.refresh(existing) - return {"report_id": existing.id, "analysis_id": analysis_id, "ready": True} - - row = create_report_row(analysis_id, path) - db.add(row) - db.commit() - db.refresh(row) - return {"report_id": row.id, "analysis_id": analysis_id, "ready": True} - - -@router.get("/{analysis_id}/download") -def download(analysis_id: int, db: Session = Depends(get_db)): - row = db.query(Report).filter(Report.analysis_id == analysis_id).first() - if not row: - raise HTTPException(status_code=404, detail="report not found — generate first") - p = Path(row.file_path) - if not p.exists(): - raise HTTPException(status_code=410, detail="report expired or missing") - return FileResponse( - path=str(p), - media_type="application/pdf", - filename=f"deepshield_report_{analysis_id}.pdf", - ) - - -@router.post("/cleanup") -def cleanup(): - n = cleanup_expired() - return {"deleted": n} diff --git a/video_service.py b/video_service.py deleted file mode 100644 index b1334fe682462f59c7eb3486c907ff033142b99e..0000000000000000000000000000000000000000 --- a/video_service.py +++ /dev/null @@ -1,151 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import List, Tuple - -import cv2 -import numpy as np -from loguru import logger -from PIL import Image - -from models.model_loader import get_model_loader -from services.image_service import classify_image - - -@dataclass -class FrameAnalysis: - index: int - timestamp_s: float - label: str - confidence: float - suspicious_prob: float # prob of the fake/manipulated class - is_suspicious: bool - has_face: bool = False - scored: bool = False # contributed to aggregate (face frames only) - - -@dataclass -class VideoAggregation: - num_frames_sampled: int - num_face_frames: int - num_suspicious_frames: int - mean_suspicious_prob: float - max_suspicious_prob: float - suspicious_ratio: float - insufficient_faces: bool - suspicious_timestamps: List[float] = field(default_factory=list) - frames: List[FrameAnalysis] = field(default_factory=list) - - -FAKE_TOKENS = ("fake", "deepfake", "manipulated", "ai", "generated", "synthetic") - - -def _is_fake_label(label: str) -> bool: - l = label.lower() - return any(tok in l for tok in FAKE_TOKENS) - - -def extract_frames(video_path: str, num_frames: int = 16) -> List[Tuple[int, float, Image.Image]]: - """Uniformly sample num_frames frames from the video. Returns list of - (frame_index, timestamp_seconds, PIL.Image). - """ - cap = cv2.VideoCapture(video_path) - if not cap.isOpened(): - raise RuntimeError(f"Failed to open video: {video_path}") - - total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0) - fps = float(cap.get(cv2.CAP_PROP_FPS) or 0.0) - if total <= 0: - cap.release() - raise RuntimeError("Video appears to have 0 frames") - - n = min(num_frames, total) - indices = np.linspace(0, max(0, total - 1), num=n, dtype=int).tolist() - - out: List[Tuple[int, float, Image.Image]] = [] - for idx in indices: - cap.set(cv2.CAP_PROP_POS_FRAMES, int(idx)) - ok, frame_bgr = cap.read() - if not ok or frame_bgr is None: - continue - frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) - pil = Image.fromarray(frame_rgb) - ts = (idx / fps) if fps > 0 else 0.0 - out.append((int(idx), float(ts), pil)) - - cap.release() - logger.info(f"Extracted {len(out)}/{n} frames from video (total={total}, fps={fps:.2f})") - return out - - -MIN_FACE_FRAMES = 3 # below this we refuse to issue a deepfake verdict - - -def _has_face(pil: Image.Image) -> bool: - detector = get_model_loader().load_face_detector() - arr = np.array(pil) - res = detector.process(arr) - return bool(getattr(res, "multi_face_landmarks", None)) - - -def classify_frames(frames: List[Tuple[int, float, Image.Image]]) -> List[FrameAnalysis]: - results: List[FrameAnalysis] = [] - for idx, ts, pil in frames: - face = _has_face(pil) - clf = classify_image(pil) - fake_prob = 0.0 - for lbl, p in clf.all_scores.items(): - if _is_fake_label(lbl): - fake_prob = max(fake_prob, float(p)) - results.append( - FrameAnalysis( - index=idx, - timestamp_s=ts, - label=clf.label, - confidence=clf.confidence, - suspicious_prob=fake_prob, - is_suspicious=(fake_prob >= 0.5) and face, - has_face=face, - scored=face, - ) - ) - return results - - -def aggregate(frames: List[FrameAnalysis]) -> VideoAggregation: - if not frames: - return VideoAggregation(0, 0, 0, 0.0, 0.0, 0.0, True) - - scored = [f for f in frames if f.scored] - num_face = len(scored) - insufficient = num_face < MIN_FACE_FRAMES - - if insufficient: - mean_p = 0.0 - max_p = 0.0 - susp_ratio = 0.0 - susp: List[FrameAnalysis] = [] - else: - probs = [f.suspicious_prob for f in scored] - susp = [f for f in scored if f.is_suspicious] - mean_p = float(np.mean(probs)) - max_p = float(np.max(probs)) - susp_ratio = len(susp) / len(scored) - - return VideoAggregation( - num_frames_sampled=len(frames), - num_face_frames=num_face, - num_suspicious_frames=len(susp), - mean_suspicious_prob=mean_p, - max_suspicious_prob=max_p, - suspicious_ratio=susp_ratio, - insufficient_faces=insufficient, - suspicious_timestamps=[round(f.timestamp_s, 2) for f in susp], - frames=frames, - ) - - -def analyze_video(video_path: str, num_frames: int = 16) -> VideoAggregation: - frames = extract_frames(video_path, num_frames=num_frames) - classified = classify_frames(frames) - return aggregate(classified) diff --git a/vlm_breakdown.py b/vlm_breakdown.py deleted file mode 100644 index 50ab212b81579d5eeeb475d0191d603fa52809ca..0000000000000000000000000000000000000000 --- a/vlm_breakdown.py +++ /dev/null @@ -1,138 +0,0 @@ -"""VLM Detailed Breakdown — Phase 14.1 - -Calls a vision-capable LLM (Gemini or OpenAI) to score 6 perceptual -components of an image for deepfake forensics. Cached per record_id. -""" -from __future__ import annotations - -import json -from io import BytesIO -from typing import Any - -from loguru import logger -from PIL import Image - -from config import settings -from schemas.common import VLMBreakdown, VLMComponentScore - -_cache: dict[str, VLMBreakdown] = {} - -_PROMPT = """\ -You are DeepShield's deepfake forensics engine. Analyze this image and score \ -each component for visual authenticity. - -Output ONLY valid JSON (no markdown fences, no extra text): -{ - "facial_symmetry": {"score": <0-100>, "notes": ""}, - "skin_texture": {"score": <0-100>, "notes": ""}, - "lighting_consistency": {"score": <0-100>, "notes": ""}, - "background_coherence": {"score": <0-100>, "notes": ""}, - "anatomy_hands_eyes": {"score": <0-100>, "notes": ""}, - "context_objects": {"score": <0-100>, "notes": ""} -} - -Scoring rules: -- 100 = perfectly natural/authentic for that component -- 0 = clear manipulation artifact for that component -- Score each independently based only on visual evidence in this image -- If a component is not visible (e.g. no hands present), score 75 and note "not visible in image" -""" - - -def _parse_response(raw: str) -> dict[str, Any]: - text = raw.strip() - if text.startswith("```"): - lines = [ln for ln in text.split("\n") if not ln.strip().startswith("```")] - text = "\n".join(lines).strip() - return json.loads(text) - - -def _to_component(d: Any) -> VLMComponentScore: - if isinstance(d, dict): - return VLMComponentScore( - score=max(0, min(100, int(d.get("score", 75)))), - notes=str(d.get("notes", ""))[:200], - ) - return VLMComponentScore() - - -def _build_breakdown(data: dict[str, Any]) -> VLMBreakdown: - return VLMBreakdown( - facial_symmetry=_to_component(data.get("facial_symmetry")), - skin_texture=_to_component(data.get("skin_texture")), - lighting_consistency=_to_component(data.get("lighting_consistency")), - background_coherence=_to_component(data.get("background_coherence")), - anatomy_hands_eyes=_to_component(data.get("anatomy_hands_eyes")), - context_objects=_to_component(data.get("context_objects")), - ) - - -def generate_vlm_breakdown( - image: Image.Image, - record_id: str | None = None, -) -> VLMBreakdown | None: - """Score 6 perceptual components via vision LLM. Returns None when unconfigured.""" - if record_id and record_id in _cache: - cached = _cache[record_id] - cached.cached = True - return cached - - if not settings.LLM_API_KEY: - logger.debug("LLM_API_KEY not set — skipping VLM breakdown") - return None - - provider = settings.LLM_PROVIDER.lower() - model_id = settings.LLM_MODEL - - try: - if provider == "openai": - breakdown = _call_openai(image, model_id) - else: - breakdown = _call_gemini(image, model_id) - - breakdown.model_used = f"{provider}/{model_id}" - if record_id: - _cache[record_id] = breakdown - - logger.info(f"VLM breakdown generated via {provider}/{model_id}") - return breakdown - - except json.JSONDecodeError as e: - logger.error(f"VLM breakdown: unparseable JSON from LLM: {e}") - return None - except Exception as e: - logger.error(f"VLM breakdown failed: {e}") - return None - - -def _call_gemini(image: Image.Image, model_id: str) -> VLMBreakdown: - import google.generativeai as genai # type: ignore - genai.configure(api_key=settings.LLM_API_KEY) - model = genai.GenerativeModel(model_id) - response = model.generate_content([_PROMPT, image]) - return _build_breakdown(_parse_response(response.text)) - - -def _call_openai(image: Image.Image, model_id: str) -> VLMBreakdown: - import base64 - from openai import OpenAI # type: ignore - - buf = BytesIO() - img = image.convert("RGB") - img.save(buf, format="JPEG", quality=85) - b64 = base64.b64encode(buf.getvalue()).decode() - - client = OpenAI(api_key=settings.LLM_API_KEY) - response = client.chat.completions.create( - model=model_id, - messages=[{ - "role": "user", - "content": [ - {"type": "text", "text": _PROMPT}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}, - ], - }], - temperature=0.2, - max_tokens=400, - ) - return _build_breakdown(_parse_response(response.choices[0].message.content))