Push

data/concept/lilac/toxicity/cohere.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:670e81b8448ab0ee5161a42b523410b3af80c6ccce8003cae78edebb9d0981c4
-size 9720631

data/concept/lilac/toxicity/openai.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e15e8235c2152b1412a8e2dee3dcb94b23e95f1fde6fb60f01b876a832e46404
-size 3678199

data/concept/lilac/toxicity/sbert.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8ac8b304760c88242eb6c567e1af87fd87731a192308df8cf43b253e24d2b0ec
-size 959111

data/datasets/local/spotify/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:32224657332b09187a737c73ab634f9d14c9ba9a240bd105f1b9819cde2afcef
-size 37128682

data/datasets/local/spotify/manifest.json

@@ -1,27 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "data_schema": {
-    "fields": {
-      "artist": {
-        "dtype": "string"
-      },
-      "song": {
-        "dtype": "string"
-      },
-      "link": {
-        "dtype": "string"
-      },
-      "text": {
-        "dtype": "string"
-      },
-      "__line_number__": {
-        "dtype": "int64"
-      },
-      "__rowid__": {
-        "dtype": "string"
-      }
-    }
-  }
-}

data/datasets/local/spotify/text/.concepts/local/outer_space/sbert-neg-100.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:39c785533ab17c040510b58b31990ba24b78646cbdfe86273973efa62ba20ebf
-size 165756

data/datasets/local/spotify/text/.concepts/local/outerspace/sbert-neg-100.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d95df038f05560b0e4429c43945eb6e773b668b805ae8f6c014529acbeeb755e
-size 185263

data/datasets/local/spotify/text/sbert/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9796beb630cc3503f3c2ac9db8f71e4c1604570836d78bbf364e801cd427c39e
-size 2709987

data/datasets/local/spotify/text/sbert/embeddings-00000-of-00001.keys.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d5df43291782b8c731d4ce56537946654c642a01dc9a4e37de394836362f6b45
-size 3727400

data/datasets/local/spotify/text/sbert/embeddings-00000-of-00001.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:94e10c23d7229541e1f60b791a659d13673b10a03649abf0ae092e0e18c5aee3
-size 170446976

data/datasets/local/spotify/text/sbert/signal_manifest.json

@@ -1,37 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "sbert(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "sbert": {
-            "repeated_field": {
-              "fields": {
-                "embedding": {
-                  "dtype": "embedding"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "signal_name": "sbert"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "sbert"
-  },
-  "enriched_path": [
-    "text"
-  ],
-  "embedding_filename_prefix": "embeddings-00000-of-00001"
-}

src/concepts/db_concept.py

@@ -324,6 +324,7 @@ class DiskConceptDB(ConceptDB):
     # Read the concepts and return a ConceptInfo containing the namespace and name.
     concept_infos = []
     for root, _, files in os.walk(self._base_dir):
+      print(root, files)
       for file in files:
         if file == CONCEPT_JSON_FILENAME:
           namespace, name = root.split('/')[-2:]

src/server.py

@@ -106,7 +106,7 @@ def startup() -> None:
     for concept in concepts:
       spaces_concept_output_dir = get_concept_output_dir(spaces_data_dir, concept.namespace,
                                                          concept.name)
-      persistent_output_dir = get_dataset_output_dir(data_path(), concept.namespace, concept.name)
+      persistent_output_dir = get_concept_output_dir(data_path(), concept.namespace, concept.name)
       shutil.rmtree(persistent_output_dir, ignore_errors=True)
       shutil.move(spaces_concept_output_dir, persistent_output_dir)