Add topic analysis

main.py

@@ -352,6 +352,14 @@ def main():
                                     href="#section53",
                                 )
                             ),
+                            Li(
+                                A(
+                                    "Topic Analysis",
+                                    href="/results#section5",
+                                    hx_get="/results#section5",
+                                    hx_target="#inner-text",
+                                )
+                            )
                         ),
                     ),
                     role="navigation",

results.py

@@ -839,6 +839,7 @@ intro_div = Div(
     Ul(
         Li("The Learning Curve of TxT360 with an Upsampling Recipe", style = "margin-bottom: 5px"),
         Li("Perplexity Analysis across time", style = "margin-bottom: 5px"),
+        Li("Topic Analysis on Data Cluster Groups", style = "margin-bottom: 5px"),
     ),
 )
 
@@ -974,6 +975,53 @@ llama_div = Div(
         ),
 )
 
+with open(os.path.join(os.path.dirname(__file__), "data", "topic_charts.json"), 'r') as f:
+    topic_charts = json.load(f)
+topic_graphs = {}
+
+for title, data in topic_charts.items():
+    if data["type"] == "barh":
+        topic_graphs[title] = go.Figure(go.Bar(
+            x=data["kwargs"]["width"],
+            y=data["kwargs"]['y'],
+            orientation='h',
+            marker_color=[
+                "rgb(" + ", ".join(str(val * 255) for val in rgb) + ')'
+                for rgb in data["kwargs"]["color"]
+            ]
+        ))
+    else:
+        topic_count_graph = go.Figure(go.Pie(
+            values=data["kwargs"]['x'],
+            labels=data["kwargs"]["labels"],
+            marker_colors=[
+                "rgb(" + ", ".join(str(val * 255) for val in rgb) + ')'
+                for rgb in data["kwargs"]["colors"]
+            ]
+        ))
+
+cluster_div = Div(
+    Section(
+        H2("Topic Analysis"),
+        P("We tried to classify data into topic groups and looked for correlations between topics and statistics of data. We believe that different groups should manifest different characteristics of distribution, which could give us some insight into the composition of dataset."),
+        H3("Methodology"),
+        P("We took the ", A("common crawl", href="https://commoncrawl.org/"), " data and clustered them into 17 topic groups using ", A("BERTopic", href="https://maartengr.github.io/BERTopic/index.html"), ". For each group, we collected and aggregated a series of metrics and calculated average scores for different quality signals and metadata."),
+        H3("Cluster Groups"),
+        P("We grouped data into the following 17 clusters"),
+        Ul(*(
+            Li(topic_name, style = "margin-bottom: 5px")
+            for topic_name in ("Arts", "Business & Economics & Finance", "Culture & Cultural geography", "Daily Life & Home & Lifestyle", "Education", "Entertainment & Travel & Hobby", "Environment", "Food & Drink & Cooking", "Health & Wellness & Medicine", "Law & Justice", "Natural Science & Formal Science & Technology", "Personal Development & Human Resources & Career", "Politics & Government", "Religion & Spirituality", "Shopping & Commodity", "Society & Social Issues & Human Rights", "Sports")
+        )),
+        H3("Results Analysis"),
+        H3("Number of document of each topic"),
+        plotly2fasthtml(topic_count_graph),
+        *(
+            Section(H3(title), plotly2fasthtml(graph))
+            for title, graph in topic_graphs.items()
+        )
+    )
+)
+
 
 def results():
     return Div(
@@ -995,6 +1043,10 @@ def results():
                 ),
                 Section(
                     llama_div,
+                ),
+                Section(
+                    cluster_div,
+                    id="section5"
                 ),
                     id="inner-text"
                 )