switch to parquet dataset

Dockerfile

@@ -8,12 +8,11 @@ ENV HOME=/code
 RUN apt install curl
 RUN pip install pip -U
 
-RUN pip install renumics-spotlight==1.2.0rc2
+RUN pip install renumics-spotlight==1.3.0rc8
 
-RUN pip install datasets
+# RUN pip install datasets
 
 COPY . .
 RUN mkdir -p /code/.cache
 RUN chmod -R 777 /code
-RUN python prepare.py
 CMD ["python", "run.py"]

cifar100-enriched-cv.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a00c5f027c054d5a697540d1396d07f70cd651ff59b6b862f81498e164279351
+size 37007568

prepare.py

@@ -1,22 +0,0 @@
-import pickle
-import datasets
-import os
-
-if __name__ == "__main__":
-    cache_file = "dataset_cache.pkl"
-    if os.path.exists(cache_file):
-        # Load dataset from cache
-        with open(cache_file, "rb") as file:
-            dataset = pickle.load(file)
-        print("Dataset loaded from cache.")
-    else:
-        # Load dataset using datasets.load_dataset()
-        dataset = datasets.load_dataset("renumics/cifar100-enriched", split="test")
-        print("Dataset loaded using datasets.load_dataset().")
-
-        # Save dataset to cache
-        with open(cache_file, "wb") as file:
-            pickle.dump(dataset, file)
-
-        print("Dataset saved to cache.")
-

run.py

@@ -2,31 +2,24 @@ import pickle
 import datasets
 from renumics import spotlight
 import os
+import pandas as pd
+import pickle 
+from renumics import spotlight
 
 if __name__ == "__main__":
     cache_file = "dataset_cache.pkl"
     issue_cache_file="sliceline.pkl"
-    if os.path.exists(cache_file):
-        # Load dataset from cache
-        with open(cache_file, "rb") as file:
-            dataset = pickle.load(file)
-        print("Dataset loaded from cache.")
-    else:
-        # Load dataset using datasets.load_dataset()
-        dataset = datasets.load_dataset("renumics/cifar100-enriched", split="train")
-        print("Dataset loaded using datasets.load_dataset().")
-
-        # Save dataset to cache
-        with open(cache_file, "wb") as file:
-            pickle.dump(dataset, file)
+    
+    import pandas as pd
 
-        print("Dataset saved to cache.")
+    #load dataframe from parquet
+    df = pd.read_parquet('cifar100-enriched-cv.parquet')    
     
     with open(issue_cache_file, "rb") as issue_file:
          issues = pickle.load(issue_file)
-
-    df = dataset.to_pandas()
+ 
     df_show = df.drop(columns=['embedding', 'probabilities'])
+    
     while True:
         view = spotlight.show(df_show, port=7860, host="0.0.0.0", issues=issues,
                     dtype={"image": spotlight.Image, "embedding_reduced": spotlight.Embedding}, allow_filebrowsing=False)