Repository: tournesol-app/tournesol

Issue:
[dev-end] load_public_dataset is broken
Hi everyone,

`load_public_dataset` seems to be broken.

Current branch:  `main` commit `c9c3b9dab`
How to reproduce:

```
$ ./run-docker-compose.sh download --user-sampling 0.1 
 ⠿ Container tournesol-dev-api    Started                                 25.9s
Waiting for api to be ready...
Waiting for api to be ready...
Waiting for api to be ready...
Waiting for api to be ready...
Waiting for api to be ready...
Waiting for api to be ready...

 Created 534 users
Created 5113 video entities
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/pandas/core/indexes/base.py", line 3621, in get_loc
    return self._engine.get_loc(casted_key)
  File "pandas/_libs/index.pyx", line 136, in pandas._libs.index.IndexEngine.get_loc
  File "pandas/_libs/index.pyx", line 163, in pandas._libs.index.IndexEngine.get_loc
  File "pandas/_libs/hashtable_class_helper.pxi", line 5198, in pandas._libs.hashtable.PyObjectHashTable.get_item
  File "pandas/_libs/hashtable_class_helper.pxi", line 5206, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 'weight'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/backend/manage.py", line 22, in <module>
    main()
  File "/backend/manage.py", line 18, in main
    execute_from_command_line(sys.argv)
  File "/usr/local/lib/python3.9/site-packages/django/core/management/__init__.py", line 446, in execute_from_command_line
    utility.execute()
  File "/usr/local/lib/python3.9/site-packages/django/core/management/__init__.py", line 440, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/usr/local/lib/python3.9/site-packages/django/core/management/base.py", line 414, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/usr/local/lib/python3.9/site-packages/django/core/management/base.py", line 460, in execute
    output = self.handle(*args, **options)
  File "/backend/tournesol/management/commands/load_public_dataset.py", line 113, in handle
    weight=values["weight"],
  File "/usr/local/lib/python3.9/site-packages/pandas/core/series.py", line 958, in __getitem__
    return self._get_value(key)
  File "/usr/local/lib/python3.9/site-packages/pandas/core/series.py", line 1069, in _get_value
    loc = self.index.get_loc(label)
  File "/usr/local/lib/python3.9/site-packages/pandas/core/indexes/base.py", line 3623, in get_loc
    raise KeyError(key) from err
```

I will investigate but any help is welcome!


Discussion:
argh probably caused by #1323 
We talked about how removing a column could break existing third-party scripts, but forgot our own scripts could also be affected :upside_down_face: 

Resolving patch:
diff --git a/backend/tournesol/management/commands/load_public_dataset.py b/backend/tournesol/management/commands/load_public_dataset.py
--- a/backend/tournesol/management/commands/load_public_dataset.py
+++ b/backend/tournesol/management/commands/load_public_dataset.py
@@ -26,6 +26,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument("--user-sampling", type=float, default=None)
+        parser.add_argument("--comparisons-url", type=str, default=PUBLIC_DATASET_URL)
 
     def create_user(self, username):
         is_pretrusted = (
@@ -55,11 +56,13 @@ def refresh_metadata(video):
 
         futures = (thread_pool.submit(refresh_metadata, video) for video in videos.values())
         for future in concurrent.futures.as_completed(futures):
-            # .result() will reraise any exception occured during refresh
+            # .result() will reraise any exception occurred during refresh
             future.result()
 
+        thread_pool.shutdown()
+
     def create_test_user(self):
-        User.objects.create_user(  # hardcoded password is delibarate # nosec B106
+        User.objects.create_user(  # hardcoded password is deliberate # nosec B106
             username="user1",
             password="tournesol",
             email="user1@tournesol.app"
@@ -68,7 +71,7 @@ def create_test_user(self):
     def handle(self, *args, **options):
         random.seed(RANDOM_SEED)
 
-        public_dataset = pd.read_csv(PUBLIC_DATASET_URL)
+        public_dataset = pd.read_csv(options["comparisons_url"])
         nb_comparisons = 0
 
         with transaction.atomic():
@@ -110,7 +113,6 @@ def handle(self, *args, **options):
                         comparison=comparison,
                         criteria=values["criteria"],
                         score=values["score"],
-                        weight=values["weight"],
                     )
                 nb_comparisons += 1
             print(f"Created {nb_comparisons} comparisons")


Test patch:
diff --git a/backend/tournesol/tests/test_load_public_dataset.py b/backend/tournesol/tests/test_load_public_dataset.py
new file mode 100644
--- /dev/null
+++ b/backend/tournesol/tests/test_load_public_dataset.py
@@ -0,0 +1,40 @@
+import tempfile
+
+from django.core.management import call_command
+from django.test import TransactionTestCase, override_settings
+from rest_framework.test import APIClient
+
+from core.models import User
+from tournesol.entities.video import TYPE_VIDEO
+from tournesol.models import Comparison, ContributorRating, Entity
+from tournesol.tests.factories.comparison import ComparisonCriteriaScoreFactory
+
+
+class TestLoadPublicDataset(TransactionTestCase):
+    serialized_rollback = True
+
+    def setUp(self):
+        self.client = APIClient()
+
+        ComparisonCriteriaScoreFactory()
+        ContributorRating.objects.update(is_public=True)
+
+        public_comparisons_resp = self.client.get("/exports/comparisons/")
+        with tempfile.NamedTemporaryFile(mode="wb", delete=False) as comparisons_file:
+            comparisons_file.write(public_comparisons_resp.content)
+            self.comparisons_path = comparisons_file.name
+
+        User.objects.all().delete()
+        Entity.objects.filter(type=TYPE_VIDEO).delete()
+
+    @override_settings(YOUTUBE_API_KEY=None)
+    def test_load_public_dataset(self):
+        self.assertEqual(User.objects.count(), 0)
+        self.assertEqual(Comparison.objects.count(), 0)
+        self.assertEqual(Entity.objects.filter(type=TYPE_VIDEO).count(), 0)
+
+        call_command("load_public_dataset", "--comparisons-url", self.comparisons_path)
+
+        self.assertEqual(User.objects.count(), 2)  # 1 user from public dataset + 1 test user
+        self.assertEqual(Comparison.objects.count(), 1)
+        self.assertEqual(Entity.objects.filter(type=TYPE_VIDEO).count(), 2)