Updated dependencies, run.py

app.ipynb

@@ -0,0 +1,83 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pickle\n",
+    "import datasets\n",
+    "from renumics import spotlight\n",
+    "from renumics.spotlight import dtypes as spotlight_dtypes\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset loaded using datasets.load_dataset().\n"
+     ]
+    }
+   ],
+   "source": [
+    "dataset_name=\"ai-aerospace/ac-text-embedding-ada-002-ams-test\"\n",
+    "dataset = datasets.load_dataset(dataset_name, split=\"train\")\n",
+    "print(\"Dataset loaded using datasets.load_dataset().\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "view = spotlight.show(dataset, port=7860, host=\"0.0.0.0\", \n",
+    "            dtype={\"used_by_questions\": spotlight_dtypes.SequenceDType(spotlight_dtypes.str_dtype)}, \n",
+    "            allow_filebrowsing=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

pyproject.toml

@@ -0,0 +1,20 @@
+[tool.poetry]
+name = "aerospace-chatbot-visualize"
+version = "0.1.0"
+description = ""
+authors = ["Dan Mueller <dsm@danmueller.pro>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.11,<3.12"
+datasets = "^2.19.0"
+ipykernel = "^6.29.4"
+renumics-spotlight = "1.6.6"
+
+
+[tool.poetry.group.dev.dependencies]
+ipykernel = "^6.29.4"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

run.py

@@ -1,30 +1,17 @@
 import pickle
 import datasets
 from renumics import spotlight
+from renumics.spotlight import dtypes as spotlight_dtypes
 import os
 
 if __name__ == "__main__":
-    cache_file = "dataset_cache.pkl"
-    if os.path.exists(cache_file):
-        # Load dataset from cache
-        with open(cache_file, "rb") as file:
-            dataset = pickle.load(file)
-        print("Dataset loaded from cache.")
-    else:
-        # Load dataset using datasets.load_dataset()
-        dataset = datasets.load_dataset("renumics/cifar100-enriched", split="train")
-        print("Dataset loaded using datasets.load_dataset().")
-
-        # Save dataset to cache
-        with open(cache_file, "wb") as file:
-            pickle.dump(dataset, file)
-
-        print("Dataset saved to cache.")
-
+    dataset_name="ai-aerospace/ac-text-embedding-ada-002-ams-test"
+    # Load dataset using datasets.load_dataset()
+    dataset = datasets.load_dataset(dataset_name, split="train")
+    print("Dataset loaded using datasets.load_dataset().")
 
     df = dataset.to_pandas()
-    df_show = df.drop(columns=['embedding', 'probabilities'])
-    while True:
-        view = spotlight.show(df_show.sample(5000, random_state=1), port=7860, host="0.0.0.0", 
-                    dtype={"image": spotlight.Image, "embedding_reduced": spotlight.Embedding}, allow_filebrowsing=False)
-        view.close()
+
+    view = spotlight.show(df, port=7860, host="0.0.0.0", 
+                dtype={"used_by_questions": spotlight_dtypes.SequenceDType(spotlight_dtypes.str_dtype)}, 
+                allow_filebrowsing=True)