refactor: update import paths and enhance problem data structure with difficulty and topic tags

backend/app/controllers/search_controller.py

@@ -1,5 +1,5 @@
-from backend.app.utils.get_embeddings import get_embedding
-from backend.app.database.find_k_nearest import get_questions_by_similarity_range
+from app.utils.get_embeddings import get_embedding
+from app.database.find_k_nearest import get_questions_by_similarity_range
 from typing import List
 
 

backend/app/database/find_k_nearest.py

@@ -1,6 +1,6 @@
 from typing import List
 # Adjust the import path as needed
-from backend.app.database.supabase_client import supabase
+from app.database.supabase_client import supabase
 
 
 def get_questions_by_similarity_range(query_embedding: List[float], page: int
@@ -14,6 +14,8 @@ def get_questions_by_similarity_range(query_embedding: List[float], page: int
             content,
             original_content,
             paid_only,
+            difficulty,
+            topictags,
             LEAST(
                 GREATEST(
                     ROUND(

backend/app/database/insert_data.py

@@ -1,6 +1,6 @@
 # insert_problems.py
 import logging
-from backend.app.database.supabase_client import supabase
+from app.database.supabase_client import supabase
 
 
 logging.basicConfig(level=logging.INFO)
@@ -8,7 +8,7 @@ logging.basicConfig(level=logging.INFO)
 
 def insert_questions(problems):
     for prob in problems:
-        # Insert into Supabase
+        prob['topicTags'] = '@'.join(prob.get('topicTags', []))
         supabase.table("problems_bge").upsert({
             "id": prob["id"],
             "title": prob["title"],
@@ -17,6 +17,8 @@ def insert_questions(problems):
             "content": prob.get("content", ""),
             "original_content": prob.get("original_content", ""),
             "embedding": prob.get("embedding", []),
-            "id_num": int(prob["id"])
+            "id_num": int(prob["id"]),
+            "difficulty": prob["difficulty"],
+            "topictags": prob["topicTags"]
         }, on_conflict=["id"]).execute()
     logging.info(f"Inserted {len(problems)} problems into Supabase.")

backend/app/database/last_fetched_data.py

@@ -1,4 +1,4 @@
-from backend.app.database.supabase_client import supabase
+from app.database.supabase_client import supabase
 
 
 def get_last_fetched_question(type):

backend/app/main.py

@@ -1,10 +1,11 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from backend.app.routes import search_route
+from app.routes import search_route
 import logging
 
-# Optional: from backend.app.scripts.populate_db import populate_db
+from app.scripts.populate_db import populate_db
 
+# Initialize FastAPI app
 app = FastAPI(title="LeetCode Vector Search API", version="1.0")
 
 # Set up logging
@@ -13,24 +14,24 @@ logging.basicConfig(level=logging.INFO)
 # Configure CORS
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=[
-        "https://leet-search-sepia.vercel.app",
-        "http://localhost:3000"
-    ],
+    # Replace with frontend URLs in production
+    allow_origins=["https://leet-search-sepia.vercel.app/search",
+                   "https://leet-search-sepia.vercel.app", "http://localhost:3000"],
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
+# Health check endpoint
+
 
 @app.get("/")
 def root():
     return {"message": "LeetCode Vector Search API is running 🚀"}
 
 
+# Register routes
 app.include_router(search_route.router, prefix="/api", tags=["Search"])
 
-if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run("backend.app.main:app", host="0.0.0.0",
-                port=7860, reload=False)
+# run initial population of the database only once then only run scraping
+# populate_db()

backend/app/routes/search_route.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter, HTTPException, Query, Body
 from typing import Dict, Any
-from backend.app.controllers.search_controller import handle_search
+from app.controllers.search_controller import handle_search
 
 router = APIRouter()
 

backend/app/scripts/populate_db.py

@@ -1,5 +1,5 @@
-from backend.app.utils.get_paid_problems import get_paid_problems
-from backend.app.utils.get_embeddings import get_embedding
+from app.utils.get_paid_problems import get_paid_problems
+from app.utils.get_embeddings import get_embedding
 import re
 import logging
 import os
@@ -61,7 +61,9 @@ def format_problem(problems=[], type=False):
             'paidOnly': type,
             'slug': problem['slug'],
             'content': clean_text,
-            'original_content': raw_html
+            'original_content': raw_html,
+            'difficulty': problem['difficulty'],
+            'topicTags': problem.get('topicTags', []),
         })
     return formatted_problems
 
@@ -75,13 +77,18 @@ def filter_problems(problems=[]):
             filtered_problems_paid.append({
                 'id': problem['questionFrontendId'],
                 'title': problem['title'],
-                'slug': problem['url'].rstrip('/').split('/')[-1]})
+                'difficulty': problem['difficulty'],
+                'slug': problem['url'].rstrip('/').split('/')[-1],
+                'topicTags': [tag['name'] for tag in problem['topicTags']],
+            })
         else:
             filtered_problems_free.append({
                 'id': problem['questionFrontendId'],
                 'title': problem['title'],
                 'slug': problem['url'].rstrip('/').split('/')[-1],
                 'content': problem['content'],
+                'difficulty': problem['difficulty'],
+                'topicTags': [tag['name'] for tag in problem['topicTags']],
             })
     return filtered_problems_free, filtered_problems_paid
 
@@ -93,7 +100,7 @@ def save_to_csv(data, filename='problems.csv'):
     csv_path = os.path.join(os.path.dirname(__file__), filename)
     with open(csv_path, 'w', newline='', encoding='utf-8') as csvfile:
         fieldnames = ['id', 'id_num', 'url', 'title',
-                      'paid_only', 'content', 'original_content', 'embedding']
+                      'paid_only', 'content', 'original_content', 'embedding', 'difficulty', 'topictags']
         writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
         writer.writeheader()
         for row in data:
@@ -104,6 +111,7 @@ def save_to_csv(data, filename='problems.csv'):
 def order_data(data):
     csv_data = []
     for problem in data:
+        problem['topictags'] = '@'.join(problem.get('topictags', []))
         csv_data.append({
             'id': problem['id'],
             'id_num': int(problem['id']),
@@ -112,7 +120,9 @@ def order_data(data):
             'paid_only': problem['paidOnly'],
             'content': problem.get('content', ''),
             'original_content': problem.get('original_content', ''),
-            'embedding': json.dumps(problem.get('embedding', []))
+            'embedding': json.dumps(problem.get('embedding', [])),
+            'difficulty': problem['difficulty'],
+            'topicTags': problem['topicTags'],
         })
     return csv_data
 

backend/app/scripts/sql.txt

@@ -13,7 +13,9 @@ CREATE TABLE problems_bge (
   paid_only BOOLEAN,
   content TEXT,
   original_content TEXT,
-  embedding vector(768)              -- Adjust dimension if needed
+  embedding vector(768),              -- Adjust dimension if needed
+  difficulty TEXT,
+  topictags TEXT
 );
 
 -- Drop pre-existing exec_sql RPC function (if exists)

backend/app/scripts/update_data.py

@@ -1,5 +1,5 @@
 import logging
-from backend.app.services.scrape_problems import scrape_problems
+from app.services.scrape_problems import scrape_problems
 
 
 def main():

backend/app/services/genrate_embeddings.py

@@ -1,5 +1,5 @@
-from backend.app.utils.get_embeddings import get_embedding
-from backend.app.database.insert_data import insert_questions
+from app.utils.get_embeddings import get_embedding
+from app.database.insert_data import insert_questions
 
 
 def generate_embeddings(data):

backend/app/services/scrape_problems.py

@@ -25,6 +25,10 @@ def get_all_problems(categorySlug="", skip=0, limit=10000, filters={}):
       paidOnly: isPaidOnly
       title
       titleSlug
+         topicTags {
+        name
+      }
+      difficulty
     }
   }
 }
@@ -53,12 +57,16 @@ def filter_problems(problems=[]):
                 'id': problem['frontendQuestionId'],
                 'title': problem['title'],
                 'slug': problem['titleSlug'],
+                'difficulty': problem['difficulty'],
+                'topicTags': [tag['name'] for tag in problem['topicTags']],
             })
         else:
             filtered_problems_free.append({
                 'id': problem['frontendQuestionId'],
                 'title': problem['title'],
                 'slug': problem['titleSlug'],
+                'difficulty': problem['difficulty'],
+                'topicTags': [tag['name'] for tag in problem['topicTags']],
             })
     return filtered_problems_free, filtered_problems_paid
 
@@ -75,6 +83,8 @@ def get_json_problem(problems=[]):
                 'title': problem['title'],
                 'slug': problem['slug'],
                 'content': response_data['content'],
+                'difficulty': problem['difficulty'],
+                'topicTags': problem.get('topicTags', []),
             })
         else:
             logging.error(
@@ -100,7 +110,9 @@ def format_problem(problems=[], type=False):
             'paidOnly': type,
             'slug': problem['slug'],
             'content': clean_text,
-            'original_content': raw_html
+            'original_content': raw_html,
+            'difficulty': problem['difficulty'],
+            'topicTags': problem.get('topicTags', []),
         })
     return formatted_problems
 

backend/app/utils/get_paid_problems.py

@@ -53,6 +53,8 @@ def get_paid_problems(problems=[]):
             'id': problem['id'],
             'title': problem['title'],
             'slug': problem['slug'],
-            'content': clean_html
+            'content': clean_html,
+            'difficulty': problem['difficulty'],
+            'topicTags': problem.get('topicTags', []),
         })
     return paid_problems