Repository: AlexsLemonade/refinebio

Issue:
Add experiment_accession column to sample metadata post-smashing
### Context

Once a dataset has been smashed, it becomes hard to tell which sample belongs to which experiment.

### Problem or idea

If the metadata for each sample included a column for the experiment accession code then it would be easier to determine this.

### Solution or next step

Add a new column to metadata for datasets for each sample that has the experiment accession code.



Discussion:

Resolving patch:
diff --git a/workers/data_refinery_workers/processors/smasher.py b/workers/data_refinery_workers/processors/smasher.py
--- a/workers/data_refinery_workers/processors/smasher.py
+++ b/workers/data_refinery_workers/processors/smasher.py
@@ -150,11 +150,13 @@ def _get_tsv_columns(samples_metadata):
                     else:
                         _add_annotation_column(annotation_columns, annotation_key)
 
-    # Return sorted columns, in which "refinebio_accession_code" is always the first,
-    # followed by the other refinebio columns (in alphabetic order), and
+    # Return sorted columns, in which "refinebio_accession_code" and "experiment_accession" are
+    # always first, followed by the other refinebio columns (in alphabetic order), and
     # annotation columns (in alphabetic order) at the end.
     refinebio_columns.discard('refinebio_accession_code')
-    return ['refinebio_accession_code'] + sorted(refinebio_columns) + sorted(annotation_columns)
+    return ['refinebio_accession_code', 'experiment_accession'] + sorted(refinebio_columns) \
+        + sorted(annotation_columns)
+
 
 def _add_annotation_value(row_data, col_name, col_value, sample_accession_code):
     """Adds a new `col_name` key whose value is `col_value` to row_data.
@@ -183,7 +185,14 @@ def _add_annotation_value(row_data, col_name, col_value, sample_accession_code):
         )
 
 
-def _get_tsv_row_data(sample_metadata):
+def _get_experiment_accession(sample_accession_code, dataset_data):
+    for experiment_accession, samples in dataset_data.items():
+        if sample_accession_code in samples:
+            return experiment_accession
+    return ""  # Should never happen, because the sample is by definition in the dataset
+
+
+def _get_tsv_row_data(sample_metadata, dataset_data):
     """Returns field values based on input sample_metadata.
 
     Some annotation fields are treated specially because they are more
@@ -245,6 +254,9 @@ def _get_tsv_row_data(sample_metadata):
                     _add_annotation_value(row_data, annotation_key, annotation_value,
                                           sample_accession_code)
 
+    row_data["experiment_accession"] = _get_experiment_accession(sample_accession_code,
+                                                                 dataset_data)
+
     return row_data
 
 
@@ -272,7 +284,7 @@ def _write_tsv_json(job_context, metadata, smash_path):
                 dw.writeheader()
                 for sample_accession_code, sample_metadata in metadata['samples'].items():
                     if sample_accession_code in experiment_data['sample_accession_codes']:
-                        row_data = _get_tsv_row_data(sample_metadata)
+                        row_data = _get_tsv_row_data(sample_metadata, job_context["dataset"].data)
                         dw.writerow(row_data)
         return tsv_paths
     # Per-Species Metadata
@@ -289,7 +301,7 @@ def _write_tsv_json(job_context, metadata, smash_path):
                 dw.writeheader()
                 for sample_metadata in metadata['samples'].values():
                     if sample_metadata.get('refinebio_organism', '') == species:
-                        row_data = _get_tsv_row_data(sample_metadata)
+                        row_data = _get_tsv_row_data(sample_metadata, job_context["dataset"].data)
                         dw.writerow(row_data)
                         samples_in_species.append(sample_metadata)
 
@@ -312,10 +324,11 @@ def _write_tsv_json(job_context, metadata, smash_path):
             dw = csv.DictWriter(tsv_file, columns, delimiter='\t')
             dw.writeheader()
             for sample_metadata in metadata['samples'].values():
-                row_data = _get_tsv_row_data(sample_metadata)
+                row_data = _get_tsv_row_data(sample_metadata, job_context["dataset"].data)
                 dw.writerow(row_data)
         return [tsv_path]
 
+
 def _quantile_normalize(job_context: Dict, ks_check=True, ks_stat=0.001) -> Dict:
     """
     Apply quantile normalization.


Test patch:
diff --git a/workers/data_refinery_workers/processors/test_smasher.py b/workers/data_refinery_workers/processors/test_smasher.py
--- a/workers/data_refinery_workers/processors/test_smasher.py
+++ b/workers/data_refinery_workers/processors/test_smasher.py
@@ -1202,7 +1202,7 @@ def setUp(self):
     @tag("smasher")
     def test_columns(self):
         columns = smasher._get_tsv_columns(self.metadata['samples'])
-        self.assertEqual(len(columns), 21)
+        self.assertEqual(len(columns), 22)
         self.assertEqual(columns[0], 'refinebio_accession_code')
         self.assertTrue('refinebio_accession_code' in columns)
         self.assertTrue('cell population' in columns)
@@ -1215,7 +1215,9 @@ def test_all_samples(self):
         """Check tsv file that includes all sample metadata."""
 
         job_context = {
-            'dataset': Dataset.objects.create(aggregate_by='ALL')
+            'dataset': Dataset.objects.create(aggregate_by='ALL',
+                                              data={'GSE56409': ['GSM1361050'],
+                                                    'E-GEOD-44719': ['E-GEOD-44719-GSM1089311']})
         }
         smasher._write_tsv_json(job_context, self.metadata, self.smash_path)
         tsv_filename = self.smash_path + "ALL/metadata_ALL.tsv"
@@ -1230,10 +1232,12 @@ def test_all_samples(self):
                     self.assertFalse('source' in row)                   # ArrayExpress specific
                     self.assertEqual(row['detection_percentage'], '98.44078')
                     self.assertEqual(row["extract"], "GSM1089311 extract 1")
+                    self.assertEqual(row["experiment_accession"], "E-GEOD-44719")
                 elif row['refinebio_accession_code'] == 'GSM1361050':
                     self.assertEqual(row['tissue'], 'Bone Marrow')      # GEO specific
                     self.assertEqual(row['refinebio_organism'], 'homo_sapiens')
                     self.assertEqual(row["contact_address"], "Crown Street")
+                    self.assertEqual(row["experiment_accession"], "GSE56409")
 
         self.assertEqual(row_num, 1)  # only two data rows in tsv file
         os.remove(tsv_filename)
@@ -1243,7 +1247,9 @@ def test_experiment(self):
         """Check tsv file that is aggregated by experiment."""
 
         job_context = {
-            'dataset': Dataset.objects.create(aggregate_by='EXPERIMENT')
+            'dataset': Dataset.objects.create(aggregate_by='EXPERIMENT',
+                                              data={'GSE56409': ['GSM1361050'],
+                                                    'E-GEOD-44719': ['E-GEOD-44719-GSM1089311']})
         }
         smasher._write_tsv_json(job_context, self.metadata, self.smash_path)
 
@@ -1254,6 +1260,7 @@ def test_experiment(self):
             reader = csv.DictReader(tsv_file, delimiter='\t')
             for row_num, row in enumerate(reader):
                 self.assertEqual(row['refinebio_accession_code'], 'E-GEOD-44719-GSM1089311')
+                self.assertEqual(row["experiment_accession"], "E-GEOD-44719")
                 self.assertEqual(row['cell population'], 'IFNa DC')  # ArrayExpress specific
                 self.assertEqual(row['dose'], '1 mL')                # ArrayExpress specific
                 self.assertEqual(row['detection_percentage'], '98.44078')
@@ -1330,7 +1337,9 @@ def test_species(self):
         """Check tsv file that is aggregated by species."""
 
         job_context = {
-            'dataset': Dataset.objects.create(aggregate_by='SPECIES'),
+            'dataset': Dataset.objects.create(aggregate_by='SPECIES',
+                                              data={'GSE56409': ['GSM1361050'],
+                                                    'E-GEOD-44719': ['E-GEOD-44719-GSM1089311']}),
             'input_files': {
                 'homo_sapiens': [], # only the key matters in this test
                 'fake_species': []  # only the key matters in this test
@@ -1347,6 +1356,7 @@ def test_species(self):
             reader = csv.DictReader(tsv_file, delimiter='\t')
             for row_num, row in enumerate(reader):
                 self.assertEqual(row['refinebio_accession_code'], 'GSM1361050')
+                self.assertEqual(row["experiment_accession"], "GSE56409")
                 self.assertEqual(row['tissue'], 'Bone Marrow')      # GEO specific
                 self.assertEqual(row['refinebio_organism'], 'homo_sapiens')
 
@@ -1371,6 +1381,7 @@ def test_species(self):
             reader = csv.DictReader(tsv_file, delimiter='\t')
             for row_num, row in enumerate(reader):
                 self.assertEqual(row['refinebio_accession_code'], 'E-GEOD-44719-GSM1089311')
+                self.assertEqual(row["experiment_accession"], "E-GEOD-44719")
                 self.assertEqual(row['cell population'], 'IFNa DC')  # ArrayExpress specific
                 self.assertEqual(row['dose'], '1 mL')                # ArrayExpress specific
                 self.assertEqual(row['detection_percentage'], '98.44078')