Repository: ImageMarkup/isic-challenge-scoring

Issue:
ValueError: Index has duplicate keys: Index(['0.12345', 'ISIC_0012345'], dtype='object', name='image')
Sentry Issue: [STADE-1S](https://sentry.io/organizations/isic/issues/1149020425/?referrer=github_integration)

```
ValueError: Index has duplicate keys: Index(['0.12345', 'ISIC_0012345'], dtype='object', name='image')
  File "core/tasks.py", line 152, in score_submission
    results = compute_metrics(truth_file, prediction_file)
  File "isic_challenge_scoring/task3.py", line 18, in compute_metrics
    prediction_probabilities = parse_csv(prediction_file_stream, categories)
  File "isic_challenge_scoring/load_csv.py", line 42, in parse_csv
    probabilities.set_index('image', drop=True, inplace=True, verify_integrity=True)
  File "pandas/core/frame.py", line 4189, in set_index
    dup=duplicates))

internal error scoring submission XXXX
```

Discussion:

Patch:
diff --git a/isic_challenge_scoring/load_csv.py b/isic_challenge_scoring/load_csv.py
index 8caef0a..d0156a1 100644
--- a/isic_challenge_scoring/load_csv.py
+++ b/isic_challenge_scoring/load_csv.py
@@ -29,7 +29,7 @@ def parse_truth_csv(csv_file_stream: TextIO) -> Tuple[pd.DataFrame, pd.DataFrame
 
 def parse_csv(csv_file_stream: TextIO, categories: pd.Index) -> pd.DataFrame:
     try:
-        probabilities = pd.read_csv(csv_file_stream, header=0)
+        probabilities = pd.read_csv(csv_file_stream, header=0, index_col=False)
     except pd.errors.ParserError as e:
         # TODO: Test this case
         raise ScoreException(f'Could not parse CSV: "{str(e)}"')
@@ -37,17 +37,28 @@ def parse_csv(csv_file_stream: TextIO, categories: pd.Index) -> pd.DataFrame:
     if 'image' not in probabilities.columns:
         raise ScoreException('Missing column in CSV: "image".')
 
+    # Pandas represents strings as 'O' (object)
+    if probabilities['image'].dtype != np.dtype('O'):
+        # Coercing to 'U' (unicode) ensures that even NaN values are converted;
+        # however, the resulting type is still 'O'
+        probabilities['image'] = probabilities['image'].astype(np.dtype('U'))
+
     probabilities['image'] = probabilities['image'].str.replace(r'\.jpg$', '', case=False)
 
-    probabilities.set_index('image', drop=True, inplace=True, verify_integrity=True)
+    if not probabilities['image'].is_unique:
+        duplicate_images = probabilities['image'][probabilities['image'].duplicated()].unique()
+        raise ScoreException(f'Duplicate image rows detected in CSV: {duplicate_images.tolist()}.')
+
+    # The duplicate check is the same as performed by 'verify_integrity'
+    probabilities.set_index('image', drop=True, inplace=True, verify_integrity=False)
 
     missing_columns = categories.difference(probabilities.columns)
     if not missing_columns.empty:
-        raise ScoreException(f'Missing columns in CSV: {list(missing_columns)}.')
+        raise ScoreException(f'Missing columns in CSV: {missing_columns.tolist()}.')
 
     extra_columns = probabilities.columns.difference(categories)
     if not extra_columns.empty:
-        raise ScoreException(f'Extra columns in CSV: {list(extra_columns)}.')
+        raise ScoreException(f'Extra columns in CSV: {extra_columns.tolist()}.')
 
     # sort by the order in categories
     probabilities = probabilities.reindex(categories, axis='columns')


Test patch:
diff --git a/tests/test_load_csv.py b/tests/test_load_csv.py
index b72384e..e81f88d 100644
--- a/tests/test_load_csv.py
+++ b/tests/test_load_csv.py
@@ -125,6 +125,26 @@ def test_parse_csv_misnamed_columns(categories):
     assert 'Missing columns in CSV: [\'DF\', \'NV\'].' == str(exc_info.value)
 
 
+def test_parse_csv_trailing_delimiters(categories):
+    prediction_file_stream = io.StringIO(
+        'image,MEL,NV,BCC,AKIEC,BKL,DF,VASC\n'
+        'ISIC_0000123,1.0,0.0,0.0,0.0,0.0,0.0,0.0,\n'
+        'ISIC_0000124,0.0,1.0,0.0,0.0,0.0,0.0,0.0,\n'
+    )
+
+    # If all data rows have trailing delimiters, 'pd.read_csv' can misinterpret the data without
+    # 'index_col=False'
+    prediction_probabilities = load_csv.parse_csv(prediction_file_stream, categories)
+
+    assert prediction_probabilities.equals(
+        pd.DataFrame(
+            [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
+            index=['ISIC_0000123', 'ISIC_0000124'],
+            columns=categories,
+        )
+    )
+
+
 def test_parse_csv_reordered_columns(categories):
     prediction_file_stream = io.StringIO(
         'NV,BCC,BKL,DF,AKIEC,MEL,VASC,image\n' '0.0,0.0,0.0,0.0,0.0,1.0,0.0,ISIC_0000123\n'
@@ -150,6 +170,17 @@ def test_parse_csv_missing_index(categories):
     assert 'Missing column in CSV: "image".' == str(exc_info.value)
 
 
+def test_parse_csv_invalid_type_index(categories):
+    prediction_file_stream = io.StringIO(
+        'image,MEL,NV,BCC,AKIEC,BKL,DF,VASC\n' '5,1.0,0.0,0.0,0.0,0.0,0.0,0.0\n'
+    )
+
+    prediction_probabilities = load_csv.parse_csv(prediction_file_stream, categories)
+
+    # Apparent numeric 'image' fields should be coerced to string / NumPy 'O'
+    assert prediction_probabilities.index.is_object()
+
+
 def test_parse_csv_missing_values(categories):
     prediction_file_stream = io.StringIO(
         'image,MEL,NV,BCC,AKIEC,BKL,DF,VASC\n'
@@ -199,6 +230,23 @@ def test_parse_csv_out_of_range_values(categories):
     )
 
 
+def test_parse_csv_duplicate_images(categories):
+    prediction_file_stream = io.StringIO(
+        'image,MEL,NV,BCC,AKIEC,BKL,DF,VASC\n'
+        'ISIC_0000123,1.0,0.0,0.0,0.0,0.0,0.0,0.0\n'
+        'ISIC_0000123,0.0,1.0,0.0,0.0,0.0,0.0,0.0\n'
+        'ISIC_0000124,0.0,0.0,1.0,0.0,0.0,0.0,0.0\n'
+        'ISIC_0000124.jpg,0.0,0.0,1.0,0.0,0.0,0.0,0.0\n'
+    )
+
+    with pytest.raises(ScoreException) as exc_info:
+        load_csv.parse_csv(prediction_file_stream, categories)
+
+    assert 'Duplicate image rows detected in CSV: [\'ISIC_0000123\', \'ISIC_0000124\'].' == str(
+        exc_info.value
+    )
+
+
 def test_validate_rows_missing_images(categories):
     truth_probabilities = pd.DataFrame(
         [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]],