Duplicate from renumics/navigate-data-issues

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+
+.venv/

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.9
+
+WORKDIR /code
+ENV HOME=/code
+
+RUN apt install curl
+RUN pip install pip -U
+
+RUN pip install renumics-spotlight==1.3.0
+
+RUN pip install datasets
+COPY prepare.py .
+RUN python prepare.py
+
+COPY . .
+RUN mkdir -p /code/.cache
+RUN chmod -R 777 /code
+CMD ["python", "run.py"]

README.md

@@ -0,0 +1,20 @@
+---
+title: Navigate Cleanlab Data Issues in CIFAR-100 with Spotlight
+emoji: 🧐
+colorFrom: gray
+colorTo: blue
+sdk: docker
+pinned: false
+license: gpl-3.0
+app_file: run.py
+datasets:
+- renumics/cifar100-enriched
+- cifar100
+tags:
+- renumics
+- spotlight
+- EDA
+duplicated_from: renumics/navigate-data-issues
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

layout.json

@@ -0,0 +1,135 @@
+{
+    "orientation": "horizontal",
+    "children": [
+        {
+            "kind": "split",
+            "weight": 44.24966799468792,
+            "orientation": "vertical",
+            "children": [
+                {
+                    "kind": "tab",
+                    "weight": 33.54784241752236,
+                    "children": [
+                        {
+                            "kind": "widget",
+                            "name": "Table",
+                            "type": "table",
+                            "config": {
+                                "tableView": "full",
+                                "visibleColumns": [
+                                    "label",
+                                    "label_score",
+                                    "near_duplicate_score",
+                                    "pred",
+                                    "outlier_score"
+                                ],
+                                "sorting": null,
+                                "orderByRelevance": false
+                            }
+                        }
+                    ]
+                },
+                {
+                    "kind": "tab",
+                    "weight": 23.686809949341544,
+                    "children": [
+                        {
+                            "kind": "widget",
+                            "name": "Issues",
+                            "type": "IssuesWidget",
+                            "config": null
+                        }
+                    ]
+                },
+                {
+                    "kind": "tab",
+                    "weight": 42.765347633136095,
+                    "children": [
+                        {
+                            "kind": "widget",
+                            "name": "Inspector",
+                            "type": "inspector",
+                            "config": {
+                                "views": [
+                                    {
+                                        "view": "ImageView",
+                                        "columns": [
+                                            "full_image"
+                                        ],
+                                        "name": "full_image",
+                                        "key": "7hA9fgoBXsKTCCFVYZfhRb"
+                                    },
+                                    {
+                                        "view": "TextLens",
+                                        "columns": [
+                                            "label"
+                                        ],
+                                        "name": "view",
+                                        "key": "a7fedf96-f36e-4836-9ffe-7b249c16db46"
+                                    },
+                                    {
+                                        "view": "TextLens",
+                                        "columns": [
+                                            "pred"
+                                        ],
+                                        "name": "view",
+                                        "key": "527a66c2-b3d4-4be0-9879-8749ee4fd0ed"
+                                    }
+                                ],
+                                "visibleColumns": 8
+                            }
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "kind": "tab",
+            "weight": 55.75033200531208,
+            "children": [
+                {
+                    "kind": "widget",
+                    "name": "Similarity Map",
+                    "type": "similaritymap",
+                    "config": {
+                        "placeBy": [
+                            "embedding_reduced"
+                        ],
+                        "reductionMethod": null,
+                        "colorBy": "label",
+                        "sizeBy": "is_label_issue",
+                        "filter": false,
+                        "umapNNeighbors": 20,
+                        "umapMetric": "cosine",
+                        "umapMinDist": 0.15,
+                        "pcaNormalization": null,
+                        "umapMenuLocalGlobalBalance": null,
+                        "umapMenuIsAdvanced": false
+                    }
+                },
+                {
+                    "kind": "widget",
+                    "name": "Scatter Plot",
+                    "type": "scatterplot",
+                    "config": {
+                        "xAxisColumn": null,
+                        "yAxisColumn": null,
+                        "colorBy": null,
+                        "sizeBy": null,
+                        "filter": false
+                    }
+                },
+                {
+                    "kind": "widget",
+                    "name": "Histogram",
+                    "type": "histogram",
+                    "config": {
+                        "columnKey": null,
+                        "stackByColumnKey": null,
+                        "filter": false
+                    }
+                }
+            ]
+        }
+    ]
+}

prepare.py

@@ -0,0 +1,47 @@
+import pickle
+import datasets
+import os
+import umap
+
+
+if __name__ == "__main__":
+    cache_file = "dataset_cache.pkl"
+    if os.path.exists(cache_file):
+        # Load dataset from cache
+        with open(cache_file, "rb") as file:
+            dataset = pickle.load(file)
+        print("Dataset loaded from cache.")
+    else:
+        # Load dataset using datasets.load_dataset()
+        ds = datasets.load_dataset("renumics/cifar100-enriched", split="test")
+
+        print("Dataset loaded using datasets.load_dataset().")
+
+        df = ds.to_pandas()
+
+
+        df = ds.rename_columns({"fine_label": "labels"}).to_pandas()
+        from tabulate import tabulate
+        from cleanlab import Datalab
+        import pandas as pd
+        import numpy as np
+
+
+        lab = Datalab(data=ds, label_name="fine_label")
+        features=np.array([x.tolist() for x in df["embedding"]])
+        pred_probs= np.array([x.tolist() for x in df["probabilities"]])
+        lab.find_issues(features=features,pred_probs=pred_probs)
+        print(tabulate(lab.get_issues().iloc[[0,1,2,3,-3,-2,-1]], headers='keys', tablefmt='psql'))
+
+
+
+        df_with_score = pd.concat([df, lab.get_issues()], axis=1)
+        df = df_with_score
+
+
+        # Save dataset to cache
+        with open(cache_file, "wb") as file:
+            pickle.dump(df, file)
+
+        print("Dataset saved to cache.")
+

requirements.txt

@@ -0,0 +1,2 @@
+renumics-spotlight
+datasets

run.py

@@ -0,0 +1,72 @@
+import pickle
+from renumics import spotlight
+import os
+import requests
+import pandas as pd
+from renumics import spotlight
+from renumics.spotlight.analysis import DataIssue
+
+
+
+if __name__ == "__main__":
+    cache_file = "dataset_cache.pkl"
+
+    if os.path.exists(cache_file):
+        # Load dataset from cache
+        with open(cache_file, "rb") as file:
+            df = pickle.load(file)
+        print("Dataset loaded from cache.")
+
+        
+        label_issue_rows = df[df["is_label_issue"]].sort_values("label_score").index.tolist()
+        label_issue = DataIssue(
+            severity="medium",
+            title="label-issue",
+            rows=label_issue_rows,
+            description="Label issue found by cleanlab - Review and correct if necessary",
+        )
+
+        outlier_issue_row = (
+            df[df["outlier_score"] < 0.6].sort_values("outlier_score").index.tolist()
+        )
+        outlier_issue = DataIssue(
+            severity="medium",
+            title="outlier-issue",
+            rows=outlier_issue_row,
+            description="Outlier score < 0.6 - Review and remove or collect more data",
+        )
+
+        near_duplicate_issue_row = (
+            df[df["is_near_duplicate_issue"]].sort_values("near_duplicate_score").index.tolist()
+        )
+        near_duplicate_issue = DataIssue(
+            severity="medium",
+            title="near-duplicate-issue",
+            rows=near_duplicate_issue_row,
+            description="Near duplicate issue found by cleanlab - Review and remove if necessary",
+        )
+
+
+        while True:
+            dtypes = {
+                "image": spotlight.Image,
+                "image_full": spotlight.Image,
+                "embedding": spotlight.Embedding,
+                "embedding_reduced": spotlight.Embedding,
+                "probabilities": spotlight.Embedding,
+            }
+
+            view = spotlight.show(
+                df.rename(columns={"fine_label_str": "label", "fine_label_prediction_str":"pred"}),
+                dtype=dtypes,
+                issues=[label_issue,outlier_issue,near_duplicate_issue],
+                layout="layout.json",
+                port=7860,
+                host="0.0.0.0",
+                allow_filebrowsing=False,
+            )
+
+            view.close()
+
+    else:
+        print(f"Dataset {cache_file} not found. Please run prepare.py first.")

spotlight-layout.json

@@ -0,0 +1 @@
+{"orientation":"vertical","children":[{"kind":"split","weight":55.52608311229001,"orientation":"horizontal","children":[{"kind":"tab","weight":41.16700201207244,"children":[{"kind":"widget","name":"Table","type":"table","config":{"tableView":"full","visibleColumns":["coarse_label_str","fine_label_prediction_str","fine_label_str","image","__idx__"],"sorting":[["__idx__","DESC"]],"orderByRelevance":false,"columnWidths":{"coarse_label":92,"coarse_label_str":165,"embedding_reduced":128,"fine_label":92,"fine_label_prediction":92,"fine_label_prediction_error":92,"fine_label_prediction_str":244,"fine_label_str":194,"full_image":256,"image":200,"split":256,"__idx__":92,"__last_edited_at__":192,"__last_edited_by__":256}}}]},{"kind":"tab","weight":33.66487769877306,"children":[{"kind":"widget","name":"Similarity Map","type":"similaritymap","config":{"placeBy":null,"reductionMethod":null,"colorBy":"fine_label_str","sizeBy":null,"filter":false,"umapNNeighbors":20,"umapMetric":null,"umapMinDist":0.15,"pcaNormalization":null,"umapMenuLocalGlobalBalance":null,"umapMenuIsAdvanced":false}},{"kind":"widget","name":"Scatter Plot","type":"scatterplot","config":{"xAxisColumn":null,"yAxisColumn":null,"colorBy":null,"sizeBy":null,"filter":false}}]},{"kind":"tab","weight":25.168120289154498,"children":[{"kind":"widget","name":"Histogram","type":"histogram","config":{"columnKey":"fine_label_str","stackByColumnKey":"fine_label_prediction_error","filter":false}}]}]},{"kind":"tab","weight":44.47391688770999,"children":[{"kind":"widget","name":"Inspector","type":"inspector","config":{"views":[{"view":"ImageView","key":"es3NP2fbYeG5TzTQqCzZKL","name":"image","columns":["image"]},{"view":"TextLens","columns":["fine_label_str"],"name":"view","key":"7262001a-1270-467c-bfe8-6b6883adbc7d"},{"view":"TextLens","columns":["fine_label_prediction_str"],"name":"view","key":"f72a8ae9-17c4-4097-b454-79a6f60865db"},{"view":"ScalarView","columns":["fine_label_prediction_error"],"name":"view","key":"be4d6afe-9093-4581-b363-df11d3c6d34a"}],"visibleColumns":4}}]}]}