Upload 3 files

README.md

@@ -1,14 +1,82 @@
----
-title: Semantic Embedding Api
-emoji: 🏆
-colorFrom: red
-colorTo: blue
-sdk: gradio
-sdk_version: 6.0.1
-app_file: app.py
-pinned: false
-license: mit
-short_description: Api semantic similarity data proposal skripsi
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Semantic Embedding API
+emoji: 🤖
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: "4.44.0"
+app_file: app.py
+pinned: false
+license: mit
+short_description: Embedding + LLM Analysis untuk deteksi kemiripan proposal
+---
+
+# 🤖 Semantic Embedding & LLM Analysis API
+
+API untuk deteksi kemiripan proposal skripsi menggunakan AI embedding dan Google Gemini.
+
+## Fitur
+
+### Embedding (Sentence Transformers)
+- **Single/Batch Embedding** - Generate embedding vektor 384 dimensi
+- **Similarity Check** - Hitung kemiripan semantik
+- **Supabase Cache** - Shared cache untuk performa
+
+### LLM Analysis (Google Gemini)
+- **Analisis Mendalam** - Reasoning seperti penilai manusia
+- **Verdict** - AMAN / PERLU_REVIEW / BERMASALAH
+- **Saran Konkret** - Rekomendasi untuk mahasiswa
+- **Auto Cache** - Hasil disimpan ke Supabase
+
+## Model & Tech
+
+| Komponen | Teknologi |
+|----------|-----------|
+| Embedding | `paraphrase-multilingual-MiniLM-L12-v2` (384 dim) |
+| LLM | Google Gemini 2.5 Pro |
+| Cache | Supabase (PostgreSQL) |
+| API | Gradio |
+
+## Required Secrets
+
+Set di **Settings > Repository secrets**:
+
+```
+SUPABASE_URL      - URL project Supabase
+SUPABASE_KEY      - Supabase anon/service key
+GEMINI_API_KEY_1  - API key Gemini #1
+GEMINI_API_KEY_2  - API key Gemini #2 (opsional)
+GEMINI_API_KEY_3  - API key Gemini #3 (opsional)
+GEMINI_API_KEY_4  - API key Gemini #4 (opsional)
+```
+
+## API Endpoints
+
+| Endpoint | Fungsi |
+|----------|--------|
+| `/get_embedding` | Single text embedding |
+| `/get_embeddings_batch` | Batch embeddings |
+| `/calculate_similarity` | Cosine similarity |
+| `/db_get_all_embeddings` | Get cached embeddings |
+| `/db_save_embedding` | Save embedding (API only) |
+| `/llm_check_status` | Check Gemini status |
+| `/llm_analyze_pair` | Full LLM analysis |
+
+## Dibuat Untuk
+
+**Monitoring Proposal Skripsi**  
+KK E (Ilmu Komputer) - Prodi Teknik Informatika  
+Universitas Komputer Indonesia (UNIKOM)
+
+🔗 [Website](https://galih-hermawan-unikom.github.io/monitoring-proksi/)
+
+## Pengembang
+
+**Galih Hermawan**  
+🌐 [galih.eu](https://galih.eu) • 🐙 [github.com/galihboy](https://github.com/galihboy) • 🐙 [github.com/Galih-Hermawan-Unikom](https://github.com/Galih-Hermawan-Unikom)
+
+📅 Terakhir diperbarui: 30 November 2025
+
+## License
+
+MIT License

app.py

@@ -4,6 +4,15 @@ import json
 import numpy as np
 import os
 import httpx
+import hashlib
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Google GenAI SDK (new library)
+from google import genai
+from google.genai import types
 
 # ==================== CONFIGURATION ====================
 
@@ -19,6 +28,83 @@ LOCAL_MODEL_PATH = r"E:\huggingface_models\hub\models--sentence-transformers--pa
 SUPABASE_URL = os.environ.get("SUPABASE_URL", "")
 SUPABASE_KEY = os.environ.get("SUPABASE_KEY", "")
 
+# Gemini API configuration with key rotation
+GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.5-pro")  # atau gemini-2.5-flash, gemini-2.5-flash-lite
+
+# Load multiple API keys for rotation
+GEMINI_API_KEYS = []
+for i in range(1, 10):  # Support up to 9 keys
+    key = os.environ.get(f"GEMINI_API_KEY_{i}", "")
+    if key:
+        GEMINI_API_KEYS.append(key)
+
+# Fallback to single key if no numbered keys found
+if not GEMINI_API_KEYS:
+    single_key = os.environ.get("GEMINI_API_KEY", "")
+    if single_key:
+        GEMINI_API_KEYS.append(single_key)
+
+# Track current key index for rotation
+current_key_index = 0
+
+def get_gemini_client():
+    """Get Gemini client with current API key"""
+    global current_key_index
+    if not GEMINI_API_KEYS:
+        return None
+    return genai.Client(api_key=GEMINI_API_KEYS[current_key_index])
+
+def rotate_api_key():
+    """Rotate to next API key"""
+    global current_key_index
+    if len(GEMINI_API_KEYS) > 1:
+        current_key_index = (current_key_index + 1) % len(GEMINI_API_KEYS)
+        print(f"🔄 Rotated to API key #{current_key_index + 1}")
+    return current_key_index
+
+def call_gemini_with_retry(prompt: str, max_retries: int = None):
+    """Call Gemini API with automatic key rotation on rate limit"""
+    global current_key_index
+    
+    if not GEMINI_API_KEYS:
+        return None, "No API keys configured"
+    
+    if max_retries is None:
+        max_retries = len(GEMINI_API_KEYS)
+    
+    last_error = None
+    
+    for attempt in range(max_retries):
+        try:
+            client = get_gemini_client()
+            response = client.models.generate_content(
+                model=GEMINI_MODEL,
+                contents=prompt
+            )
+            return response, None
+            
+        except Exception as e:
+            error_str = str(e).lower()
+            last_error = str(e)
+            
+            # Check if rate limit error
+            if "429" in error_str or "rate" in error_str or "quota" in error_str or "resource" in error_str:
+                print(f"⚠️ Rate limit hit on key #{current_key_index + 1}: {e}")
+                rotate_api_key()
+                continue
+            else:
+                # Non-rate-limit error, don't retry
+                return None, str(e)
+    
+    return None, f"All API keys exhausted. Last error: {last_error}"
+
+# Initialize and print status
+if GEMINI_API_KEYS:
+    print(f"✅ Gemini configured with {len(GEMINI_API_KEYS)} API key(s)")
+    print(f"   Model: {GEMINI_MODEL}")
+else:
+    print("⚠️ No Gemini API keys found")
+
 def get_model_path():
     """Deteksi environment dan return path model yang sesuai"""
     # Cek apakah folder lokal ada
@@ -212,6 +298,269 @@ def db_check_connection():
         return {"connected": False, "error": str(e)}
 
 
+# ==================== LLM CACHE FUNCTIONS (SUPABASE) ====================
+
+def db_get_llm_analysis(pair_hash: str):
+    """Ambil cached LLM analysis dari Supabase by pair_hash"""
+    if not SUPABASE_URL or not SUPABASE_KEY:
+        return None
+    
+    try:
+        url = f"{SUPABASE_URL}/rest/v1/llm_analysis?pair_hash=eq.{pair_hash}&select=*"
+        
+        with httpx.Client(timeout=10.0) as client:
+            response = client.get(url, headers=get_supabase_headers())
+        
+        if response.status_code == 200:
+            data = response.json()
+            if data and len(data) > 0:
+                result = data[0]
+                # Parse similar_aspects from JSONB
+                if isinstance(result.get('similar_aspects'), str):
+                    result['similar_aspects'] = json.loads(result['similar_aspects'])
+                result['from_cache'] = True
+                return result
+        return None
+    except Exception as e:
+        print(f"Error getting cached LLM analysis: {e}")
+        return None
+
+
+def db_save_llm_analysis(pair_hash: str, proposal1_judul: str, proposal2_judul: str, result: dict):
+    """Simpan LLM analysis result ke Supabase"""
+    if not SUPABASE_URL or not SUPABASE_KEY:
+        return False
+    
+    try:
+        url = f"{SUPABASE_URL}/rest/v1/llm_analysis"
+        headers = get_supabase_headers()
+        headers["Prefer"] = "resolution=merge-duplicates"  # Upsert
+        
+        payload = {
+            "pair_hash": pair_hash,
+            "proposal1_judul": proposal1_judul[:500] if proposal1_judul else "",
+            "proposal2_judul": proposal2_judul[:500] if proposal2_judul else "",
+            "similarity_score": result.get("similarity_score"),
+            "verdict": result.get("verdict"),
+            "reasoning": result.get("reasoning"),
+            "saran": result.get("saran"),
+            "similar_aspects": json.dumps(result.get("similar_aspects", {})),
+            "differentiator": result.get("differentiator"),
+            "model_used": result.get("model_used", GEMINI_MODEL)
+        }
+        
+        with httpx.Client(timeout=10.0) as client:
+            response = client.post(url, headers=headers, json=payload)
+        
+        if response.status_code in [200, 201]:
+            print(f"✅ LLM result cached: {pair_hash[:8]}...")
+            return True
+        else:
+            print(f"⚠️ Failed to cache LLM result: {response.status_code}")
+            return False
+    except Exception as e:
+        print(f"Error saving LLM analysis: {e}")
+        return False
+
+
+# ==================== LLM FUNCTIONS (GEMINI) ====================
+
+def generate_pair_hash(proposal1: dict, proposal2: dict) -> str:
+    """Generate unique hash untuk pasangan proposal"""
+    def proposal_hash(p):
+        content = f"{p.get('nim', '')}|{p.get('judul', '')}|{p.get('deskripsi', '')}|{p.get('problem', '')}|{p.get('metode', '')}"
+        return hashlib.md5(content.encode()).hexdigest()[:16]
+    
+    h1 = proposal_hash(proposal1)
+    h2 = proposal_hash(proposal2)
+    # Sort untuk konsistensi (A,B = B,A)
+    sorted_hashes = sorted([h1, h2])
+    return hashlib.md5(f"{sorted_hashes[0]}|{sorted_hashes[1]}".encode()).hexdigest()[:32]
+
+
+def llm_analyze_pair(proposal1_json: str, proposal2_json: str, use_cache: bool = True):
+    """Analisis kemiripan dua proposal menggunakan Gemini LLM"""
+    if not GEMINI_API_KEYS:
+        return {"error": "Gemini API key not configured. Set GEMINI_API_KEY_1, GEMINI_API_KEY_2, etc in .env file"}
+    
+    try:
+        proposal1 = json.loads(proposal1_json)
+        proposal2 = json.loads(proposal2_json)
+    except json.JSONDecodeError:
+        return {"error": "Invalid JSON format for proposals"}
+    
+    # Generate pair hash untuk caching
+    pair_hash = generate_pair_hash(proposal1, proposal2)
+    
+    # Check cache first
+    if use_cache:
+        cached_result = db_get_llm_analysis(pair_hash)
+        if cached_result:
+            print(f"📦 Using cached LLM result: {pair_hash[:8]}...")
+            return cached_result
+    
+    # Build prompt
+    prompt = f"""Anda adalah penilai kemiripan proposal skripsi yang ahli dan berpengalaman. Analisis dua proposal berikut dengan KRITERIA AKADEMIK yang benar.
+
+ATURAN PENILAIAN PENTING:
+1. Proposal skripsi dianggap BERMASALAH hanya jika KETIGA aspek ini SAMA: Topik/Domain + Dataset/Objek Penelitian + Metode/Algoritma
+2. Jika METODE BERBEDA (walaupun topik & dataset sama) → AMAN, karena memberikan kontribusi ilmiah berbeda
+3. Jika DATASET/OBJEK BERBEDA (walaupun topik & metode sama) → AMAN, karena studi kasus berbeda
+4. Jika TOPIK/DOMAIN BERBEDA → AMAN
+5. Penelitian replikasi dengan variasi adalah HAL YANG WAJAR dalam dunia akademik
+
+PROPOSAL 1:
+- NIM: {proposal1.get('nim', 'N/A')}
+- Nama: {proposal1.get('nama', 'N/A')}
+- Judul: {proposal1.get('judul', 'N/A')}
+- Deskripsi: {proposal1.get('deskripsi', 'N/A')[:500] if proposal1.get('deskripsi') else 'N/A'}
+- Problem Statement: {proposal1.get('problem', 'N/A')[:500] if proposal1.get('problem') else 'N/A'}
+- Metode: {proposal1.get('metode', 'N/A')}
+
+PROPOSAL 2:
+- NIM: {proposal2.get('nim', 'N/A')}
+- Nama: {proposal2.get('nama', 'N/A')}
+- Judul: {proposal2.get('judul', 'N/A')}
+- Deskripsi: {proposal2.get('deskripsi', 'N/A')[:500] if proposal2.get('deskripsi') else 'N/A'}
+- Problem Statement: {proposal2.get('problem', 'N/A')[:500] if proposal2.get('problem') else 'N/A'}
+- Metode: {proposal2.get('metode', 'N/A')}
+
+ANALISIS dengan cermat, lalu berikan output JSON (HANYA JSON, tanpa markdown):
+{{
+    "similarity_score": <0-100, tinggi HANYA jika topik+dataset+metode SEMUA sama>,
+    "verdict": "<BERMASALAH jika score>=80, PERLU_REVIEW jika 50-79, AMAN jika <50>",
+    "similar_aspects": {{
+        "topik": <true/false - apakah tema/domain penelitian sama>,
+        "dataset": <true/false - apakah objek/data penelitian sama>,
+        "metode": <true/false - apakah algoritma/metode sama>,
+        "pendekatan": <true/false - apakah framework/pendekatan sama>
+    }},
+    "differentiator": "<aspek pembeda utama: metode/dataset/domain/tidak_ada>",
+    "reasoning": "<analisis mendalam 4-5 kalimat: jelaskan persamaan dan perbedaan dari aspek topik, dataset, dan metode. Jelaskan mengapa proposal ini aman/bermasalah berdasarkan kriteria akademik>",
+    "saran": "<nasihat konstruktif 2-3 kalimat untuk mahasiswa: jika aman, beri saran penguatan diferensiasi. Jika bermasalah, beri warning dan alternatif arah penelitian>"
+}}"""
+
+    # Call Gemini API with retry/rotation
+    response, error = call_gemini_with_retry(prompt)
+    
+    if error:
+        return {"error": f"Gemini API error: {error}"}
+    
+    try:
+        # Parse response
+        response_text = response.text.strip()
+        
+        # Clean response (remove markdown code blocks if present)
+        if response_text.startswith("```"):
+            lines = response_text.split("\n")
+            response_text = "\n".join(lines[1:-1])  # Remove first and last lines
+        
+        result = json.loads(response_text)
+        result["pair_hash"] = pair_hash
+        result["model_used"] = GEMINI_MODEL
+        result["api_key_used"] = current_key_index + 1
+        result["from_cache"] = False
+        
+        # Save to cache
+        db_save_llm_analysis(
+            pair_hash=pair_hash,
+            proposal1_judul=proposal1.get('judul', ''),
+            proposal2_judul=proposal2.get('judul', ''),
+            result=result
+        )
+        
+        return result
+        
+    except json.JSONDecodeError as e:
+        return {
+            "error": "Failed to parse LLM response as JSON",
+            "raw_response": response_text if 'response_text' in dir() else "No response",
+            "parse_error": str(e)
+        }
+
+
+def llm_check_status():
+    """Check Gemini API status"""
+    if not GEMINI_API_KEYS:
+        return {
+            "configured": False,
+            "error": "No GEMINI_API_KEY found in environment"
+        }
+    
+    response, error = call_gemini_with_retry("Respond with only: OK")
+    
+    if error:
+        return {
+            "configured": True,
+            "total_keys": len(GEMINI_API_KEYS),
+            "model": GEMINI_MODEL,
+            "status": "error",
+            "error": error
+        }
+    
+    return {
+        "configured": True,
+        "total_keys": len(GEMINI_API_KEYS),
+        "current_key": current_key_index + 1,
+        "model": GEMINI_MODEL,
+        "status": "connected",
+        "test_response": response.text.strip()[:50]
+    }
+
+
+def llm_analyze_simple(judul1: str, judul2: str, metode1: str, metode2: str):
+    """Simplified analysis - hanya judul dan metode (untuk testing cepat)"""
+    if not GEMINI_API_KEYS:
+        return {"error": "Gemini API key not configured"}
+    
+    prompt = f"""Anda adalah penilai kemiripan proposal skripsi yang ahli. Bandingkan dua proposal berikut dengan KRITERIA AKADEMIK yang benar.
+
+ATURAN PENILAIAN PENTING:
+1. Proposal skripsi dianggap BERMASALAH hanya jika KETIGA aspek ini SAMA: Topik/Domain + Dataset + Metode
+2. Jika METODE BERBEDA (walaupun topik sama) → AMAN, karena kontribusi berbeda
+3. Jika DATASET BERBEDA (walaupun topik & metode sama) → AMAN, karena studi kasus berbeda
+4. Jika TOPIK/DOMAIN BERBEDA → AMAN
+
+Proposal 1:
+- Judul: {judul1}
+- Metode: {metode1}
+
+Proposal 2:
+- Judul: {judul2}
+- Metode: {metode2}
+
+ANALISIS dengan cermat, lalu berikan output JSON (HANYA JSON, tanpa markdown):
+{{
+    "similarity_score": <0-100, tinggi HANYA jika topik+dataset+metode SEMUA sama>,
+    "verdict": "<BERMASALAH jika score>=80, PERLU_REVIEW jika 50-79, AMAN jika <50>",
+    "topik_sama": <true/false>,
+    "metode_sama": <true/false>,
+    "differentiator": "<aspek pembeda utama: metode/dataset/domain/tidak_ada>",
+    "reasoning": "<analisis mendalam 3-4 kalimat: jelaskan persamaan, perbedaan, dan mengapa aman/bermasalah>",
+    "saran": "<nasihat konstruktif untuk mahasiswa, misal: cara memperkuat diferensiasi, atau warning jika terlalu mirip>"
+}}"""
+
+    response, error = call_gemini_with_retry(prompt)
+    
+    if error:
+        return {"error": error}
+    
+    try:
+        response_text = response.text.strip()
+        
+        if response_text.startswith("```"):
+            lines = response_text.split("\n")
+            response_text = "\n".join(lines[1:-1])
+        
+        result = json.loads(response_text)
+        result["model_used"] = GEMINI_MODEL
+        result["api_key_used"] = current_key_index + 1
+        return result
+        
+    except json.JSONDecodeError as e:
+        return {"error": f"Failed to parse response: {e}", "raw": response_text}
+
+
 # Gradio Interface
 with gr.Blocks(title="Semantic Embedding API") as demo:
     gr.Markdown("# 🔤 Semantic Embedding API")
@@ -252,6 +601,7 @@ with gr.Blocks(title="Semantic Embedding API") as demo:
     with gr.Tab("💾 Database (Supabase)"):
         gr.Markdown("### Supabase Cache Operations")
         gr.Markdown("Proxy untuk akses Supabase (API key aman di server)")
+        gr.Markdown("*Note: Operasi write (save) hanya tersedia melalui API untuk keamanan.*")
         
         with gr.Row():
             db_check_btn = gr.Button("🔌 Check Connection", variant="secondary")
@@ -274,18 +624,72 @@ with gr.Blocks(title="Semantic Embedding API") as demo:
         db_get_btn = gr.Button("🔍 Get Embedding", variant="primary")
         db_get_output = gr.JSON(label="Embedding Result")
         db_get_btn.click(fn=db_get_embedding, inputs=[db_nim_input, db_hash_input], outputs=db_get_output)
+    
+    with gr.Tab("🤖 LLM Analysis (Gemini)"):
+        gr.Markdown("### Analisis Kemiripan dengan LLM")
+        gr.Markdown("Menggunakan Google Gemini untuk analisis mendalam dengan penjelasan")
+        
+        with gr.Row():
+            llm_check_btn = gr.Button("🔌 Check Gemini Status", variant="secondary")
+            llm_check_output = gr.JSON(label="Gemini Status")
+            llm_check_btn.click(fn=llm_check_status, outputs=llm_check_output)
         
         gr.Markdown("---")
         
-        gr.Markdown("#### Save Embedding")
-        db_save_input = gr.Textbox(
-            label="Embedding Data (JSON)", 
-            placeholder='{"nim": "123", "content_hash": "abc", "embedding_combined": [...], ...}',
-            lines=4
+        gr.Markdown("#### Quick Analysis (Judul + Metode saja)")
+        with gr.Row():
+            with gr.Column():
+                llm_judul1 = gr.Textbox(label="Judul Proposal 1", placeholder="Analisis Sentimen dengan SVM...", lines=2)
+                llm_metode1 = gr.Textbox(label="Metode 1", placeholder="Support Vector Machine")
+            with gr.Column():
+                llm_judul2 = gr.Textbox(label="Judul Proposal 2", placeholder="Klasifikasi Sentimen dengan SVM...", lines=2)
+                llm_metode2 = gr.Textbox(label="Metode 2", placeholder="Support Vector Machine")
+        
+        llm_simple_btn = gr.Button("🚀 Analyze (Quick)", variant="primary")
+        llm_simple_output = gr.JSON(label="Quick Analysis Result")
+        llm_simple_btn.click(
+            fn=llm_analyze_simple, 
+            inputs=[llm_judul1, llm_judul2, llm_metode1, llm_metode2], 
+            outputs=llm_simple_output
+        )
+        
+        gr.Markdown("---")
+        
+        gr.Markdown("#### Full Analysis (Complete Proposal Data)")
+        gr.Markdown("*Hasil di-cache ke Supabase. Request yang sama akan menggunakan cache.*")
+        with gr.Row():
+            llm_proposal1 = gr.Textbox(
+                label="Proposal 1 (JSON)", 
+                placeholder='{"nim": "123", "nama": "Ahmad", "judul": "...", "deskripsi": "...", "problem": "...", "metode": "..."}',
+                lines=5
+            )
+            llm_proposal2 = gr.Textbox(
+                label="Proposal 2 (JSON)", 
+                placeholder='{"nim": "456", "nama": "Budi", "judul": "...", "deskripsi": "...", "problem": "...", "metode": "..."}',
+                lines=5
+            )
+        
+        with gr.Row():
+            llm_use_cache = gr.Checkbox(label="Gunakan Cache", value=True, info="Uncheck untuk force refresh dari Gemini")
+            llm_full_btn = gr.Button("🔍 Analyze (Full)", variant="primary")
+        
+        llm_full_output = gr.JSON(label="Full Analysis Result")
+        llm_full_btn.click(
+            fn=llm_analyze_pair, 
+            inputs=[llm_proposal1, llm_proposal2, llm_use_cache], 
+            outputs=llm_full_output
         )
-        db_save_btn = gr.Button("💾 Save Embedding", variant="primary")
-        db_save_output = gr.JSON(label="Save Result")
-        db_save_btn.click(fn=db_save_embedding, inputs=db_save_input, outputs=db_save_output)
+        
+        gr.Markdown("""
+        **Output mencakup:**
+        - `similarity_score`: Skor 0-100 (tinggi hanya jika topik+dataset+metode sama)
+        - `verdict`: BERMASALAH / PERLU_REVIEW / AMAN
+        - `reasoning`: Analisis mendalam dari AI
+        - `similar_aspects`: Aspek yang mirip (topik/dataset/metode/pendekatan)
+        - `differentiator`: Pembeda utama
+        - `saran`: Nasihat untuk mahasiswa
+        - `from_cache`: true jika hasil dari cache
+        """)
     
     with gr.Accordion("📡 API Usage (untuk Developer)", open=False):
         gr.Markdown("""

requirements.txt

@@ -3,3 +3,5 @@ sentence-transformers>=2.2.0
 torch
 numpy
 httpx>=0.24.0
+google-genai>=1.0.0
+python-dotenv>=1.0.0