update likes and downloads

app.py

@@ -142,15 +142,15 @@ async def get_all_new_models():
 
 async def sync_data():
     print("Fetching models")
-    new_models = await get_all_new_models()
-    print(f"Found {len(new_models)} models")
+    all_models = await get_all_new_models()
+    print(f"Found {len(all_models)} models")
     # save list of all models for ids
     with open(DB_FOLDER / "models.json", "w") as f:
-        json.dump(new_models, f)
+        json.dump(all_models, f)
     # with open(DB_FOLDER / "models.json", "r") as f:
     #     new_models = json.load(f)
 
-    new_models_ids = [model['id'] for model in new_models]
+    new_models_ids = [model['id'] for model in all_models]
 
     # get existing models
     with database.get_db() as db:
@@ -159,23 +159,39 @@ async def sync_data():
         existing_models = [row['id'] for row in cursor.fetchall()]
     models_ids_to_add = list(set(new_models_ids) - set(existing_models))
     # find all models id to add from new_models
-    models = [model for model in new_models if model['id'] in models_ids_to_add]
+    models = [model for model in all_models if model['id'] in models_ids_to_add]
 
     print(f"Found {len(models)} new models")
     for model in tqdm(models):
         model_id = model['id']
+        likes = model['likes']
+        downloads = model['downloads']
         model_card = fetch_model_card(model_id)
         images = await find_image_in_model_card(model_card)
         classifier = run_classifier(images)
         # update model row with image and classifier data
         with database.get_db() as db:
             cursor = db.cursor()
-            cursor.execute("INSERT INTO models(id, data) VALUES (?, ?)",
-                           [model_id, json.dumps({
-                               **model,
-                               "images": images,
-                               "class": classifier
-                           })])
+            cursor.execute("INSERT INTO models(id, data, likes, downloads) VALUES (?, ?, ?, ?)",
+                           [model_id,
+                            json.dumps({
+                                **model,
+                                "images": images,
+                                "class": classifier
+                            }),
+                            likes,
+                            downloads
+                            ])
+            db.commit()
+    print("Update likes and downloads")
+    for model in tqdm(all_models):
+        model_id = model['id']
+        likes = model['likes']
+        downloads = model['downloads']
+        with database.get_db() as db:
+            cursor = db.cursor()
+            cursor.execute("UPDATE models SET likes = ?, downloads = ? WHERE id = ?",
+                           [likes, downloads, model_id])
             db.commit()
 
     if (len(models) > 0):
@@ -214,27 +230,34 @@ class Sort(str, Enum):
 def get_page(page: int = 1, sort: Sort = Sort.trending):
     page = page if page > 0 else 1
     if sort == Sort.trending:
-        sort_query = "json_extract(data, '$.likes') / MYPOWER((JULIANDAY('now') - JULIANDAY(datetime(json_extract(data, '$.lastModified')))) + 2, 2) DESC"
+        sort_query = "((likes + downloads)/2) / MYPOWER((JULIANDAY('now') - JULIANDAY(datetime(json_extract(data, '$.lastModified')))) + 2, 1.5) DESC"
     elif sort == Sort.recent:
         sort_query = "datetime(json_extract(data, '$.lastModified')) DESC"
     elif sort == Sort.likes:
-        sort_query = "json_extract(data, '$.likes') DESC"
+        sort_query = "likes DESC"
 
     with database.get_db() as db:
         cursor = db.cursor()
         cursor.execute(f"""
             SELECT *, COUNT(*) OVER() AS total
             FROM models
-            WHERE json_extract(data, '$.likes') > 4
+            WHERE likes > 4
             ORDER BY {sort_query}
             LIMIT {MAX_PAGE_SIZE} OFFSET {(page - 1) * MAX_PAGE_SIZE}
         """)
         results = cursor.fetchall()
         total = results[0]['total'] if results else 0
         total_pages = (total + MAX_PAGE_SIZE - 1) // MAX_PAGE_SIZE
+        models_data = []
+        for result in results:
+            data = json.loads(result['data'])
+            # update downloads and likes from db table
+            data['downloads'] = result['downloads']
+            data['likes'] = result['likes']
+            models_data.append(data)
 
     return {
-        "models": [json.loads(result['data']) for result in results],
+        "models": models_data,
         "totalPages": total_pages
     }
 
@@ -244,8 +267,8 @@ def read_root():
     return "Just a bot to sync data from diffusers gallery"
 
 
-# @app.on_event("startup")
-# @repeat_every(seconds=60 * 60 * 24, wait_first=True)
-# async def repeat_sync():
-#     await sync_data()
-#     return "Synced data to huggingface datasets"
+@app.on_event("startup")
+@repeat_every(seconds=60 * 60 * 24, wait_first=False)
+async def repeat_sync():
+    await sync_data()
+    return "Synced data to huggingface datasets"

schema.sql

@@ -5,6 +5,8 @@ BEGIN TRANSACTION;
 CREATE TABLE models (
     id TEXT PRIMARY KEY NOT NULL,
     data json,
+    likes INTEGER DEFAULT 0 NOT NULL,
+    downloads INTEGER DEFAULT 0 NOT NULL,
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
 );