|
|
|
import os |
|
import pandas as pd |
|
|
|
import nomic |
|
from nomic import atlas |
|
import numpy as np |
|
|
|
NOMIC_KEY = os.getenv('NOMIC_KEY') |
|
nomic.login(NOMIC_KEY) |
|
|
|
|
|
def build_nomic(dataset): |
|
df = dataset['train'].to_pandas() |
|
|
|
non_embedding_columns = ['date_utc', 'title', 'flair', 'content', 'poster', 'permalink', 'id', 'content_length', |
|
'score', 'percentile_ranges'] |
|
|
|
|
|
percentiles = df['score'].quantile([0, .1, .2, .3, .4, .5, .6, .7, .8, .9]).tolist() |
|
|
|
|
|
bins = sorted(set(percentiles + [df['score'].max()])) |
|
|
|
|
|
|
|
labels = [int(i * 10) for i in range(len(bins) - 1)] |
|
|
|
|
|
|
|
df['percentile_ranges'] = pd.cut(df['score'], bins=bins, labels=labels, include_lowest=True) |
|
|
|
|
|
project = atlas.map_data(embeddings=np.stack(df['embedding'].values), |
|
data=df[non_embedding_columns].to_dict(orient='records'), |
|
id_field='id', |
|
identifier='BORU Subreddit Neural Search', |
|
) |