app

app.py

@@ -30,15 +30,18 @@ question_types = sorted(set(ex["question_type"] for ex in fact_dataset))
 
 def get_stats():
     total_examples = Counter(ex["question_type"] for ex in fact_dataset)
-    curated_examples = Counter(row["question_type"] for row in examples())
-    return {
+    curated_examples = Counter(row["question_type"] for row in examples.rows)
+    stats = {
         qt: {"total": total_examples[qt], "curated": curated_examples[qt]}
         for qt in question_types
     }
+    return stats
 
 
 def get_example(selected_type=None):
     evaluated_ids = set(row["example_id"] for row in examples())
+    print(f"completed: {evaluated_ids}")
+
     available_examples = [
         ex for ex in fact_dataset if ex["example_id"] not in evaluated_ids
     ]
@@ -46,9 +49,12 @@ def get_example(selected_type=None):
         available_examples = [
             ex for ex in available_examples if ex["question_type"] == selected_type
         ]
+
     if not available_examples:
         return None
+
     example = random.choice(available_examples)
+
     keep_keys = [
         "example_id",
         "question_type",
@@ -64,40 +70,28 @@ def get_example(selected_type=None):
         "accuracy_oracle",
         "accuracy_status",
     ]
-    return {k: example[k] for k in keep_keys if k in example}
-
 
-def upload_to_hf():
-    create_repo(
-        "rbiswasfc/iclr-eval-examples",
-        token=os.environ.get("HF_TOKEN"),
-        private=True,
-        repo_type="dataset",
-        exist_ok=True,
-    )
-    annotations = examples()
-    hf_ds = Dataset.from_list(annotations)
-    hf_ds.push_to_hub("rbiswasfc/iclr-eval-examples", token=os.environ.get("HF_TOKEN"))
+    return {k: example[k] for k in keep_keys if k in example}
 
 
+# app
 style = Style("""
-    body { background-color: #1e1e1e; color: #d4d4d4; font-family: Arial, sans-serif; }
-    h1, h2, h3 { color: #61dafb; }
-    .example-container { margin-top: 20px; }
-    .example-table { border-collapse: collapse; width: 100%; }
-    .example-table th, .example-table td { border: 1px solid #3a3a3a; padding: 8px; text-align: left; }
-    .example-table th { background-color: #2a2a2a; color: #61dafb; }
-    .example-table td { color: #d4d4d4; }
-    #evaluation-form { margin-top: 20px; }
-    #evaluation-form button { margin-right: 10px; background-color: #0e639c; color: white; border: none; padding: 10px 20px; cursor: pointer; }
-    #evaluation-form button:hover { background-color: #1177bb; }
-    select { background-color: #2a2a2a; color: #d4d4d4; border: 1px solid #3a3a3a; padding: 5px; }
-    a { color: #61dafb; text-decoration: none; }
-    a:hover { text-decoration: underline; }
-""")
-
-app = FastHTML(hdrs=(style,))
-rt = app.route
+            body { background-color: #1e1e1e; color: #d4d4d4; font-family: Arial, sans-serif; }
+            h1, h2, h3 { color: #61dafb; }
+            .example-container { margin-top: 20px; }
+            .example-table { border-collapse: collapse; width: 100%; }
+            .example-table th, .example-table td { border: 1px solid #3a3a3a; padding: 8px; text-align: left; }
+            .example-table th { background-color: #2a2a2a; color: #61dafb; }
+            .example-table td { color: #d4d4d4; }
+            #evaluation-form { margin-top: 20px; }
+            #evaluation-form button { margin-right: 10px; background-color: #0e639c; color: white; border: none; padding: 10px 20px; cursor: pointer; }
+            #evaluation-form button:hover { background-color: #1177bb; }
+            select { background-color: #2a2a2a; color: #d4d4d4; border: 1px solid #3a3a3a; padding: 5px; }
+            a { color: #61dafb; text-decoration: none; }
+            a:hover { text-decoration: underline; }
+        """)
+
+app, rt = fast_app(hdrs=(style,))
 
 
 def render_stats(stats):
@@ -128,8 +122,24 @@ def render_example(example):
     )
 
 
+def upload_to_hf():
+    create_repo(
+        repo_id="rbiswasfc/iclr-eval-examples",
+        token=os.environ.get("HF_TOKEN"),
+        private=True,
+        repo_type="dataset",
+        exist_ok=True,
+    )
+
+    examples = db.t.examples
+    annotations = examples()
+
+    hf_ds = Dataset.from_list(annotations)
+    hf_ds.push_to_hub("rbiswasfc/iclr-eval-examples", token=os.environ.get("HF_TOKEN"))
+
+
 @rt("/")
-def get(question_type: str = None):
+async def get(question_type: str = None):
     stats = get_stats()
     example = get_example(question_type)
 
@@ -142,10 +152,12 @@ def get(question_type: str = None):
         hx_push_url="true",
     )
 
-    content = (
-        Div(H2("All examples of this type have been evaluated!"), render_stats(stats))
-        if example is None
-        else Div(
+    if example is None:
+        content = Div(
+            H2("All examples of this type have been evaluated!"), render_stats(stats)
+        )
+    else:
+        content = Div(
             H2("Example"),
             Div(
                 render_example(example),
@@ -155,23 +167,22 @@ def get(question_type: str = None):
                         name="decision",
                         value="good",
                         hx_post="/evaluate",
-                        hx_target="#example-container",
+                        hx_target="#example-details",
                     ),
                     Button(
                         "Bad Example",
                         name="decision",
                         value="bad",
                         hx_post="/evaluate",
-                        hx_target="#example-container",
+                        hx_target="#example-details",
                     ),
+                    # Hidden(name="example", value=json.dumps(example)),
                     Hidden(name="example_id", value=str(example["example_id"])),
-                    Hidden(name="question_type", value=example["question_type"]),
                     id="evaluation-form",
                 ),
                 id="example-container",
             ),
         )
-    )
 
     view_stats_link = A("Curation Stats", href="/stats", cls="view-stats-link")
 
@@ -185,13 +196,14 @@ def get(question_type: str = None):
 
 
 @rt("/stats")
-def get():
+async def get():
     stats = get_stats()
-    stats_table = render_stats(stats)
+    stats = render_stats(stats)
+
     return Titled(
         "Curation Statistics",
         Div(
-            stats_table,
+            stats,
             A("Back to Curation", href="/", cls="back-link"),
             cls="container",
         ),
@@ -199,52 +211,28 @@ def get():
 
 
 @rt("/evaluate")
-def post(decision: str, example_id: str, question_type: str):
+async def post(decision: str, example_id: str):
     example_id = int(example_id)
-    example = next((ex for ex in fact_dataset if ex["example_id"] == example_id), None)
-    if example:
-        examples.insert(
-            {
-                "id": len(examples()) + 1,
-                "example_id": example_id,
-                "question_type": question_type,
-                "question": example["question"],
-                "answer": example["answer"],
-                "decision": decision,
-            }
-        )
-        upload_to_hf()
+    example_dict = fact_dataset[example_id]
+
+    # Insert the evaluated example into the database
+    examples.insert(
+        {
+            "id": len(list(examples.rows)) + 1,  # Auto-increment ID
+            "example_id": example_dict["example_id"],
+            "question_type": example_dict["question_type"],
+            "question": example_dict["question"],
+            "answer": example_dict["answer"],
+            "decision": decision,
+        }
+    )
+    upload_to_hf()
+    new_example = get_example(example_dict["question_type"])
 
-    new_example = get_example(question_type)
     if new_example is None:
-        return Div(
-            H2("All examples of this type have been evaluated!"),
-            render_stats(get_stats()),
-        )
+        return Div(H2("All examples of this type have been evaluated!"))
     else:
-        return Div(
-            render_example(new_example),
-            Form(
-                Button(
-                    "Good Example",
-                    name="decision",
-                    value="good",
-                    hx_post="/evaluate",
-                    hx_target="#example-container",
-                ),
-                Button(
-                    "Bad Example",
-                    name="decision",
-                    value="bad",
-                    hx_post="/evaluate",
-                    hx_target="#example-container",
-                ),
-                Hidden(name="example_id", value=str(new_example["example_id"])),
-                Hidden(name="question_type", value=new_example["question_type"]),
-                id="evaluation-form",
-            ),
-            id="example-container",
-        )
+        return render_example(new_example)
 
 
 # serve()
@@ -253,4 +241,5 @@ if __name__ == "__main__":
 
     import uvicorn
 
+    # setup_hf_backup(app)
     uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", 7860)))