Adding nomic

app.py

@@ -4,9 +4,10 @@ from pathlib import Path
 import gradio as gr
 from huggingface_hub import WebhookPayload, WebhooksServer
 
-from src.utilities import load_datasets, merge_and_update_datasets
 from src.my_logger import setup_logger
+from src.utilities import load_datasets, merge_and_update_datasets
 from src.visualize_logs import log_file_to_html_string
+from src.build_nomic import build_nomic
 
 proj_dir = Path(__name__).parent
 
@@ -14,7 +15,7 @@ logger = setup_logger(__name__)
 
 SUBREDDIT = os.environ["SUBREDDIT"]
 USERNAME = os.environ["USERNAME"]
-OG_DATASET= f"{USERNAME}/dataset-creator-reddit-{SUBREDDIT}"
+OG_DATASET = f"{USERNAME}/dataset-creator-reddit-{SUBREDDIT}"
 PROCESSED_DATASET = os.environ['PROCESSED_DATASET']
 HUGGINGFACE_AUTH_TOKEN = os.environ["HUGGINGFACE_AUTH_TOKEN"]
 WEBHOOK_SECRET = os.getenv("HF_WEBHOOK_SECRET", 'secret')
@@ -50,10 +51,14 @@ async def community(payload: WebhookPayload):
     dataset = merge_and_update_datasets(dataset, original_dataset)
 
     # Push the augmented dataset to the Hugging Face hub
-    logger.debug(f"Pushing processed data to the Hugging Face Hub...")
+    logger.info(f"Pushing processed data to the Hugging Face Hub...")
     dataset.push_to_hub(PROCESSED_DATASET, token=HUGGINGFACE_AUTH_TOKEN)
     logger.info(f"Pushed processed data to the Hugging Face Hub")
 
+    logger.info(f"Building Nomic...")
+    build_nomic(dataset=dataset)
+    logger.info(f"Built Nomic")
+
 if __name__ == '__main__':
     app.launch(server_name="0.0.0.0", show_error=True, server_port=7860)
     # ui.queue().launch(server_name="0.0.0.0", show_error=True, server_port=7860)

src/build_nomic.py

@@ -0,0 +1,38 @@
+# https://atlas.nomic.ai/data/derek2/boru-subreddit-neural-search/map
+import os
+import pandas as pd
+
+import nomic
+from nomic import atlas
+import numpy as np
+
+NOMIC_KEY = os.getenv('NOMIC_KEY')
+nomic.login(NOMIC_KEY)
+
+
+def build_nomic(dataset):
+    df = dataset['train'].to_pandas()
+
+    non_embedding_columns = ['date_utc', 'title', 'flair', 'content', 'poster', 'permalink', 'id', 'content_length',
+                             'score', 'percentile_ranges']
+
+    # Calculate the 0th, 10th, 20th, ..., 90th percentiles for the 'score' column
+    percentiles = df['score'].quantile([0, .1, .2, .3, .4, .5, .6, .7, .8, .9]).tolist()
+
+    # Ensure the bins are unique and include the maximum score
+    bins = sorted(set(percentiles + [df['score'].max()]))
+
+    # Define the labels for the percentile ranges
+    # The number of labels should be one less than the number of bins
+    labels = [int(i * 10) for i in range(len(bins) - 1)]
+
+    # Add a 'percentile_ranges' column to the DataFrame
+    # This assigns each score to its corresponding percentile range
+    df['percentile_ranges'] = pd.cut(df['score'], bins=bins, labels=labels, include_lowest=True)
+
+    # Create Atlas project
+    project = atlas.map_data(embeddings=np.stack(df['embedding'].values),
+                             data=df[non_embedding_columns].to_dict(orient='records'),
+                             id_field='id',
+                             identifier='BORU Subreddit Neural Search',
+                             )