Repository: hodcroftlab/covariants
Pull request #569: 523 defining mutations file format
Author: login: AdvancedCodingMonkey
Created: 2025-03-27T13:37:13Z | Merged: 2025-04-10T08:01:31Z

This PR prepares a script for merging the the auto-generated and hand-curated defining mutations files. In doing so, it also defines the data format that will be used for the defining mutations.

The data format can be inspected in `tests/data/defining_mutations/expected_output/`


Diff:
diff --git a/.gitignore b/.gitignore
index 90f46d0a40..24f798755e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ __pycache__/
 compare_lineages/output_files/
 information_files/
 /data/
+tests/data/defining_mutations/output/
 
 #figures with dates
 figures/overall_trends-202*
diff --git a/requirements.txt b/requirements.txt
index 925415bdc5..8ea5381ba9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,8 @@
 matplotlib==3.5.2
 numpy==1.22.4
 pandas==1.4.2
+polars==1.24.0
 python-dotenv==0.20.0
 requests==2.32.2
 scipy==1.8.1
 seaborn==0.11.2
-
-polars==1.24.0
\ No newline at end of file
diff --git a/scripts/merge_defining_mutations.py b/scripts/merge_defining_mutations.py
new file mode 100644
index 0000000000..2abff0e84d
--- /dev/null
+++ b/scripts/merge_defining_mutations.py
@@ -0,0 +1,390 @@
+import json
+import logging
+import os
+from dataclasses import dataclass
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+@dataclass
+class Mutation:
+    symbol_from: str
+    position: int
+    symbol_to: str
+
+    @classmethod
+    def parse_mutation_string(cls, mutation_string: str) -> "Mutation":
+        return Mutation(symbol_from=mutation_string[0], position=int(mutation_string[1:-1]), symbol_to=mutation_string[-1])
+
+    @property
+    def affected_genes(self):
+        genes = []
+        for gene, bounds in GENE_BOUNDS.items():
+            if bounds[0] <= self.position <= bounds[1]:
+                genes.append(gene)
+        return genes
+
+    @property
+    def positions_on_genes(self):
+        return [int(np.floor((self.position - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in self.affected_genes]
+
+    def to_dict(self):
+        return {'ref': self.symbol_from, 'pos': self.position, 'alt': self.symbol_to}
+
+
+
+@dataclass
+class AminoAcidMutation(Mutation):
+    gene: str
+
+    @classmethod
+    def parse_amino_acid_string(cls, amino_acid_mutations_string: str) -> "AminoAcidMutation":
+        gene, aa =amino_acid_mutations_string.split(':')
+        mut = cls.parse_mutation_string(aa)
+        return AminoAcidMutation(mut.symbol_from, mut.position, mut.symbol_to, gene)
+
+    def to_string(self) -> str:
+        return ''.join([self.gene, ':', self.symbol_from, str(self.position), self.symbol_to])
+
+    def to_dict(self):
+        return {'gene': self.gene, 'ref': self.symbol_from, 'pos': self.position, 'alt': self.symbol_to}
+
+
+def match_nuc_to_aas(nuc: str | None, aas: list[str]) -> str | None:
+    if not nuc:
+        return None
+    nuc_obj = Mutation.parse_mutation_string(nuc)
+    if len(nuc_obj.affected_genes) == 0:
+        return None
+    aas_obj = [AminoAcidMutation.parse_amino_acid_string(aa) for aa in aas]
+    for aa in aas_obj:
+        if aa.gene in nuc_obj.affected_genes and aa.position in nuc_obj.positions_on_genes:
+            return aa.to_string()
+
+
+def replace_list_of_empty_string(y: str | list | np.ndarray) -> str | list | np.ndarray:
+    if len(y) == 1 and y[0] == '':
+            return []
+    return y
+
+
+def process_auto_generated_data(mutations: pl.DataFrame):
+    empty_strings_removed = mutations.to_pandas().applymap(replace_list_of_empty_string)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    deletions_to_ranges = (
+        empty_strings_removed[deletion_columns]
+        .applymap(
+            lambda deletion_cell: [[int(pos) for pos in del_range.split('-')] for del_range in deletion_cell])
+        .applymap(
+            lambda deletion_cell: [list(range(del_range[0], del_range[1] + 1)) if len(del_range) > 1 else del_range for del_range in
+                                   deletion_cell])
+        .applymap(lambda x: list(pd.core.common.flatten(x)))
+    )
+    expand_deletions = empty_strings_removed.assign(**{col: deletions_to_ranges[col] for col in deletion_columns})
+
+    # TODO: clarify if using 'X' here is the correct way to work around the missing nucleotide in auto-generated deletions: https://github.com/hodcroftlab/covariants/issues/577
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_mutations = reformat_deletions.with_columns(
+        nuc_mutation_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_mutation_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_mutation_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_mutation_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    match_aa_mutations = pl.struct('nuc_mutation', 'aa_mutation').map_elements(
+        lambda mutation_row: match_nuc_to_aas(mutation_row['nuc_mutation'], mutation_row['aa_mutation']), pl.String
+    )
+    wuhan = (
+        combine_nuc_mutations
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_mutation=pl.col('nuc_mutation_wuhan'),
+            aa_mutation=pl.col('aa_mutation_wuhan'))
+        .explode('nuc_mutation')
+        .with_columns(
+            aa_mutation=match_aa_mutations,
+            reference=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_mutations
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_mutation='nuc_mutation_pango_parent',
+            aa_mutation=pl.col('aa_mutation_pango_parent'))
+        .explode('nuc_mutation')
+        .with_columns(
+            aa_mutation=match_aa_mutations,
+            reference=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_reference_column_is_filled(output)
+
+    return output
+
+
+def load_auto_generated_data(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        'lineage',
+        'unaliased',
+        'parent',
+        'children',
+        'nextstrain_clade',
+        'designation_date')
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    return lineages, mutations
+
+
+def process_hand_curated_file(path: str) -> pl.DataFrame:
+    filename = os.path.basename(path)
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            reference=pl.lit('wuhan')
+        )
+        .rename({'aa_change': 'aa_mutation', 'nuc_change': 'nuc_mutation'})
+        .select('nextstrain_clade', 'nuc_mutation', 'aa_mutation', 'reference', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            reference=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_reference_column_is_filled(output)
+
+    return output
+
+
+def check_reference_column_is_filled(df: pl.DataFrame) -> None:
+    if not df.filter(pl.col('reference').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("reference").is_null())}')
+
+
+def import_mutation_data(hand_curated_data_dir: str, auto_generated_data_dir: str) -> tuple[
+    pl.DataFrame, pl.DataFrame, pl.DataFrame]:
+
+    lineages, auto_generated_mutations_raw = load_auto_generated_data(os.path.join(auto_generated_data_dir, 'auto_generated.json'))
+    auto_generated_mutations = process_auto_generated_data(auto_generated_mutations_raw)
+
+    hand_curated_mutations = (
+        import_hand_curated_data(hand_curated_data_dir)
+        .join(lineages.select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+    )
+
+    return lineages, hand_curated_mutations, auto_generated_mutations
+
+
+def import_hand_curated_data(hand_curated_data_dir: str) -> pl.DataFrame:
+    hand_curated_data_files = os.listdir(hand_curated_data_dir)
+    hand_curated_data = pl.concat([
+        process_hand_curated_file(os.path.join(hand_curated_data_dir, hand_curated_file)) for hand_curated_file in
+        hand_curated_data_files
+    ])
+
+    return hand_curated_data
+
+
+def merge_mutation_data(hand_curated_mutations: pl.DataFrame, auto_generated_mutations: pl.DataFrame) -> pl.DataFrame:
+    hand_curated_raw = hand_curated_mutations.with_columns(nuc_mutation_raw=pl.col('nuc_mutation').str.slice(1))
+    auto_generated_raw = auto_generated_mutations.with_columns(nuc_mutation_raw=pl.col('nuc_mutation').str.slice(1))
+    combined = (auto_generated_raw
+                .join(hand_curated_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_mutation_raw', 'reference'],
+                      how='full',
+                      suffix='_hand_curated',
+                      coalesce=True)
+                .drop('nuc_mutation_raw'))
+
+    not_in_hand_curated = len(
+        auto_generated_raw.join(hand_curated_raw, on=['lineage', 'nextstrain_clade', 'nuc_mutation_raw', 'reference'],
+                                how='anti'))
+    not_in_auto_generated = len(
+        hand_curated_raw.join(auto_generated_raw, on=['lineage', 'nextstrain_clade', 'nuc_mutation_raw', 'reference'],
+                              how='anti'))
+    if not_in_hand_curated > 0 or not_in_auto_generated > 0:
+        logging.info(f'unmatched mutations: hand-curated {not_in_auto_generated}, auto-generated {not_in_hand_curated}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_mutation=pl.coalesce(pl.col('aa_mutation_hand_curated'), pl.col('aa_mutation')),
+            nuc_mutation=pl.coalesce(pl.col('nuc_mutation_hand_curated'), pl.col('nuc_mutation')))
+        .drop('aa_mutation_hand_curated', 'nuc_mutation_hand_curated')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_mutation').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'reference', 'mutation_type', 'aa_mutation', 'nuc_mutation', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'reference', 'mutation_type', 'aa_mutation')
+        .agg(pl.col('nuc_mutation'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_mutation').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ]).rename({'nuc_mutation': 'nuc_mutations'})
+
+    return grouped_by_aa
+
+
+def reformat_df_to_dicts(merged: pl.DataFrame, lineages: pl.DataFrame) -> dict:
+    aggregate_mutations = (
+        merged
+        .group_by('lineage', 'nextstrain_clade', 'reference', 'mutation_type')
+        .agg(pl.struct('aa_mutation', 'nuc_mutations', 'notes').alias('mutations'))
+    )
+
+    pivot_mutation_types = (
+        aggregate_mutations
+        .pivot('mutation_type', values='mutations')
+        .select('lineage', 'nextstrain_clade', mutation_type=pl.struct('reference', 'coding', 'silent'))
+        .rename({'mutation_type': 'mutations'})
+        .group_by('lineage', 'nextstrain_clade').agg('mutations')
+    )
+
+    add_lineages = pivot_mutation_types.join(lineages, on=['lineage', 'nextstrain_clade'])
+
+    output = add_lineages.select('lineage',
+                                 'nextstrain_clade',
+                                 'mutations',
+                                 'children',
+                                 'designation_date',
+                                 'unaliased',
+                                 'parent')
+
+    output_dicts = output.to_dicts()
+    for lineage in output_dicts:
+        for mutations in lineage['mutations']:
+            for mutation_type in ['coding', 'silent']:
+                mutations[mutation_type] = mutations.get(mutation_type) or []
+                for mutation in mutations[mutation_type]:
+                    if mutation_type == 'coding':
+                        mutation.update({'nuc_mutations': [Mutation.parse_mutation_string(mut).to_dict() for mut in mutation['nuc_mutations']]})
+                        mutation.update({'aa_mutation': AminoAcidMutation.parse_amino_acid_string(mutation['aa_mutation']).to_dict()})
+                    else:
+                        mutation.update({'nuc_mutation': Mutation.parse_mutation_string(mutation['nuc_mutations'][0]).to_dict()})
+                        mutation.pop('nuc_mutations')
+                        mutation.pop('aa_mutation')
+                    if not mutation['notes']:
+                        mutation.pop('notes')
+
+    return output_dicts
+
+
+def save_mutations_to_file(output: dict, output_dir: str):
+    for lineage in output:
+        with open(os.path.join(output_dir, f'{lineage["lineage"]}.json'), 'w') as f:
+            json.dump(lineage, f)
+
+def save_lineages_to_file(lineages: pl.DataFrame, output_dir: str):
+    clusters = {'clusters': lineages.select('lineage', 'nextstrain_clade').to_dicts()}
+    with open(os.path.join(output_dir, 'definingMutationsClusters.json'), 'w') as f:
+        json.dump(clusters, f, indent=2)
+
+
+def main(hand_curated_data_dir='../tests/data/defining_mutations/emma',
+         auto_generated_data_dir='../tests/data/defining_mutations/auto_generated',
+         output_dir='../tests/data/defining_mutations/output'):
+    lineages, hand_curated_mutations, auto_generated_mutations = import_mutation_data(hand_curated_data_dir,
+                                                                                      auto_generated_data_dir)
+
+    merged_mutations = merge_mutation_data(hand_curated_mutations, auto_generated_mutations)
+
+    output = reformat_df_to_dicts(merged_mutations, lineages)
+
+    save_mutations_to_file(output, output_dir)
+    save_lineages_to_file(lineages, output_dir)
+    # TODO: nextclade parent: https://github.com/hodcroftlab/covariants/issues/582
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/data/defining_mutations/auto_generated/auto_generated.json b/tests/data/defining_mutations/auto_generated/auto_generated.json
new file mode 100644
index 0000000000..727f0eb08f
--- /dev/null
+++ b/tests/data/defining_mutations/auto_generated/auto_generated.json
@@ -0,0 +1,403 @@
+{
+ "BQ.1": {
+  "lineage": "BQ.1",
+  "unaliased": "B.1.1.529.5.3.1.1.1.1.1",
+  "parent": "BE.1.1.1",
+  "children": [
+   "BQ.1.1",
+   "BQ.1.2",
+   "BQ.1.3",
+   "BQ.1.4",
+   "BQ.1.5",
+   "BQ.1.6",
+   "BQ.1.7",
+   "BQ.1.8",
+   "BQ.1.9",
+   "BQ.1.10",
+   "BQ.1.11",
+   "BQ.1.12",
+   "BQ.1.13",
+   "BQ.1.14",
+   "BQ.1.15",
+   "BQ.1.16",
+   "BQ.1.17",
+   "BQ.1.18",
+   "BQ.1.19",
+   "BQ.1.20",
+   "BQ.1.21",
+   "BQ.1.22",
+   "BQ.1.23",
+   "BQ.1.24",
+   "BQ.1.25",
+   "BQ.1.26",
+   "BQ.1.27",
+   "BQ.1.28",
+   "BQ.1.29",
+   "BQ.1.30",
+   "BQ.1.31",
+   "BQ.1.32",
+   "BQ.1.33"
+  ],
+  "nextstrainClade": "22E",
+  "nucSubstitutions": [
+   "C241T",
+   "T670G",
+   "C1931A",
+   "C2790T",
+   "T2954C",
+   "C3037T",
+   "G4184A",
+   "C4321T",
+   "C9344T",
+   "A9424G",
+   "C9534T",
+   "C10029T",
+   "C10198T",
+   "G10447A",
+   "C10449A",
+   "C11750T",
+   "G12160A",
+   "C12880T",
+   "T14257C",
+   "C14408T",
+   "C15714T",
+   "G16935A",
+   "C17410T",
+   "A18163G",
+   "C19955T",
+   "A20055G",
+   "C21618T",
+   "G21987A",
+   "T22200G",
+   "G22578A",
+   "C22674T",
+   "T22679C",
+   "C22686T",
+   "A22688G",
+   "G22775A",
+   "A22786C",
+   "G22813T",
+   "T22882G",
+   "A22893C",
+   "T22917G",
+   "T22942A",
+   "G22992A",
+   "C22995A",
+   "A23013C",
+   "T23018G",
+   "A23055G",
+   "A23063T",
+   "T23075C",
+   "A23403G",
+   "C23525T",
+   "T23599G",
+   "C23604A",
+   "C23854A",
+   "G23948T",
+   "A24424T",
+   "T24469A",
+   "C25000T",
+   "C25584T",
+   "C26060T",
+   "C26270T",
+   "G26529A",
+   "C26577G",
+   "G26709A",
+   "C27807T",
+   "C27889T",
+   "A28271T",
+   "C28311T",
+   "C28312T",
+   "G28681T",
+   "G28881A",
+   "G28882A",
+   "G28883C",
+   "A29510C",
+   "G29868A"
+  ],
+  "aaSubstitutions": [
+   "E:T9I",
+   "M:D3N",
+   "M:Q19E",
+   "M:A63T",
+   "N:P13L",
+   "N:E136D",
+   "N:R203K",
+   "N:G204R",
+   "N:S413R",
+   "ORF1a:S135R",
+   "ORF1a:Q556K",
+   "ORF1a:T842I",
+   "ORF1a:G1307S",
+   "ORF1a:L3027F",
+   "ORF1a:T3090I",
+   "ORF1a:T3255I",
+   "ORF1a:P3395H",
+   "ORF1a:L3829F",
+   "ORF1b:Y264H",
+   "ORF1b:P314L",
+   "ORF1b:M1156I",
+   "ORF1b:R1315C",
+   "ORF1b:I1566V",
+   "ORF1b:T2163I",
+   "ORF3a:T223I",
+   "ORF9b:P10F",
+   "S:T19I",
+   "S:A27S",
+   "S:G142D",
+   "S:V213G",
+   "S:G339D",
+   "S:S371F",
+   "S:S373P",
+   "S:S375F",
+   "S:T376A",
+   "S:D405N",
+   "S:R408S",
+   "S:K417N",
+   "S:N440K",
+   "S:K444T",
+   "S:L452R",
+   "S:N460K",
+   "S:S477N",
+   "S:T478K",
+   "S:E484A",
+   "S:F486V",
+   "S:Q498R",
+   "S:N501Y",
+   "S:Y505H",
+   "S:D614G",
+   "S:H655Y",
+   "S:N679K",
+   "S:P681H",
+   "S:N764K",
+   "S:D796Y",
+   "S:Q954H",
+   "S:N969K"
+  ],
+  "nucDeletions": [
+   "11288-11296",
+   "21633-21641",
+   "21765-21770",
+   "28362-28370",
+   "29734-29759"
+  ],
+  "aaDeletions": [
+   "N:E31-",
+   "N:R32-",
+   "N:S33-",
+   "ORF1a:S3675-",
+   "ORF1a:G3676-",
+   "ORF1a:F3677-",
+   "ORF9b:E27-",
+   "ORF9b:N28-",
+   "ORF9b:A29-",
+   "S:L24-",
+   "S:P25-",
+   "S:P26-",
+   "S:H69-",
+   "S:V70-"
+  ],
+  "nucSubstitutionsNew": [
+   "T14257C",
+   "T22942A"
+  ],
+  "aaSubstitutionsNew": [
+   "ORF1b:Y264H",
+   "S:N460K"
+  ],
+  "nucDeletionsNew": [],
+  "aaDeletionsNew": [],
+  "nucSubstitutionsReverted": [],
+  "aaSubstitutionsReverted": [],
+  "nucDeletionsReverted": [],
+  "aaDeletionsReverted": [],
+  "frameShifts": [
+   ""
+  ],
+  "designationDate": "2022-09-02"
+ },
+ "BE.1.1.1": {
+  "lineage": "BE.1.1.1",
+  "unaliased": "B.1.1.529.5.3.1.1.1.1",
+  "parent": "BE.1.1",
+  "children": [
+   "BQ.1",
+   "BQ.2"
+  ],
+  "nextstrainClade": "22B",
+  "nucSubstitutions": [
+   "C241T",
+   "T670G",
+   "C1931A",
+   "C2790T",
+   "T2954C",
+   "C3037T",
+   "G4184A",
+   "C4321T",
+   "C9344T",
+   "A9424G",
+   "C9534T",
+   "C10029T",
+   "C10198T",
+   "G10447A",
+   "C10449A",
+   "C11750T",
+   "G12160A",
+   "C12880T",
+   "C14408T",
+   "C15714T",
+   "G16935A",
+   "C17410T",
+   "A18163G",
+   "C19955T",
+   "A20055G",
+   "C21618T",
+   "G21987A",
+   "T22200G",
+   "G22578A",
+   "C22674T",
+   "T22679C",
+   "C22686T",
+   "A22688G",
+   "G22775A",
+   "A22786C",
+   "G22813T",
+   "T22882G",
+   "A22893C",
+   "T22917G",
+   "G22992A",
+   "C22995A",
+   "A23013C",
+   "T23018G",
+   "A23055G",
+   "A23063T",
+   "T23075C",
+   "A23403G",
+   "C23525T",
+   "T23599G",
+   "C23604A",
+   "C23854A",
+   "G23948T",
+   "A24424T",
+   "T24469A",
+   "C25000T",
+   "C25584T",
+   "C26060T",
+   "C26270T",
+   "G26529A",
+   "C26577G",
+   "G26709A",
+   "C27807T",
+   "C27889T",
+   "A28271T",
+   "C28311T",
+   "C28312T",
+   "G28681T",
+   "G28881A",
+   "G28882A",
+   "G28883C",
+   "A29510C",
+   "G29868A"
+  ],
+  "aaSubstitutions": [
+   "E:T9I",
+   "M:D3N",
+   "M:Q19E",
+   "M:A63T",
+   "N:P13L",
+   "N:E136D",
+   "N:R203K",
+   "N:G204R",
+   "N:S413R",
+   "ORF1a:S135R",
+   "ORF1a:Q556K",
+   "ORF1a:T842I",
+   "ORF1a:G1307S",
+   "ORF1a:L3027F",
+   "ORF1a:T3090I",
+   "ORF1a:T3255I",
+   "ORF1a:P3395H",
+   "ORF1a:L3829F",
+   "ORF1b:P314L",
+   "ORF1b:M1156I",
+   "ORF1b:R1315C",
+   "ORF1b:I1566V",
+   "ORF1b:T2163I",
+   "ORF3a:T223I",
+   "ORF9b:P10F",
+   "S:T19I",
+   "S:A27S",
+   "S:G142D",
+   "S:V213G",
+   "S:G339D",
+   "S:S371F",
+   "S:S373P",
+   "S:S375F",
+   "S:T376A",
+   "S:D405N",
+   "S:R408S",
+   "S:K417N",
+   "S:N440K",
+   "S:K444T",
+   "S:L452R",
+   "S:S477N",
+   "S:T478K",
+   "S:E484A",
+   "S:F486V",
+   "S:Q498R",
+   "S:N501Y",
+   "S:Y505H",
+   "S:D614G",
+   "S:H655Y",
+   "S:N679K",
+   "S:P681H",
+   "S:N764K",
+   "S:D796Y",
+   "S:Q954H",
+   "S:N969K"
+  ],
+  "nucDeletions": [
+   "11288-11296",
+   "21633-21641",
+   "21765-21770",
+   "28362-28370",
+   "29734-29759"
+  ],
+  "aaDeletions": [
+   "N:E31-",
+   "N:R32-",
+   "N:S33-",
+   "ORF1a:S3675-",
+   "ORF1a:G3676-",
+   "ORF1a:F3677-",
+   "ORF9b:E27-",
+   "ORF9b:N28-",
+   "ORF9b:A29-",
+   "S:L24-",
+   "S:P25-",
+   "S:P26-",
+   "S:H69-",
+   "S:V70-"
+  ],
+  "nucSubstitutionsNew": [
+   "A22893C",
+   "C28312T"
+  ],
+  "aaSubstitutionsNew": [
+   "ORF9b:P10F",
+   "S:K444T"
+  ],
+  "nucDeletionsNew": [],
+  "aaDeletionsNew": [],
+  "nucSubstitutionsReverted": [],
+  "aaSubstitutionsReverted": [
+   "ORF9b:P10S"
+  ],
+  "nucDeletionsReverted": [],
+  "aaDeletionsReverted": [],
+  "frameShifts": [
+   ""
+  ],
+  "designationDate": "2022-08-24"
+ }
+}
\ No newline at end of file
diff --git a/tests/data/defining_mutations/auto_generated/auto_generated_edge_cases.json b/tests/data/defining_mutations/auto_generated/auto_generated_edge_cases.json
new file mode 100644
index 0000000000..e1019f266d
--- /dev/null
+++ b/tests/data/defining_mutations/auto_generated/auto_generated_edge_cases.json
@@ -0,0 +1,518 @@
+{
+  "ContainsListsWithEmptyStrings": {
+    "lineage": "B",
+    "unaliased": "B",
+    "parent": "",
+    "children": [
+      "B.1",
+      "B.3",
+      "B.4",
+      "B.5",
+      "B.6",
+      "B.10",
+      "B.11",
+      "B.12",
+      "B.13",
+      "B.15",
+      "B.18",
+      "B.19",
+      "B.20",
+      "B.23",
+      "B.26",
+      "B.27",
+      "B.28",
+      "B.29",
+      "B.30",
+      "B.31",
+      "B.32",
+      "B.33",
+      "B.34",
+      "B.35",
+      "B.36",
+      "B.37",
+      "B.38",
+      "B.39",
+      "B.40",
+      "B.41",
+      "B.42",
+      "B.43",
+      "B.44",
+      "B.45",
+      "B.46",
+      "B.47",
+      "B.49",
+      "B.50",
+      "B.51",
+      "B.52",
+      "B.53",
+      "B.55",
+      "B.56",
+      "B.57",
+      "B.58",
+      "B.60",
+      "B.61"
+    ],
+    "nextstrainClade": "19A",
+    "nucSubstitutions": [
+      ""
+    ],
+    "aaSubstitutions": [
+      ""
+    ],
+    "nucDeletions": [
+      ""
+    ],
+    "aaDeletions": [
+      ""
+    ],
+    "nucSubstitutionsNew": [
+      ""
+    ],
+    "aaSubstitutionsNew": [
+      ""
+    ],
+    "nucDeletionsNew": [
+      ""
+    ],
+    "aaDeletionsNew": [
+      ""
+    ],
+    "nucSubstitutionsReverted": [],
+    "aaSubstitutionsReverted": [],
+    "nucDeletionsReverted": [],
+    "aaDeletionsReverted": [],
+    "frameShifts": [
+      ""
+    ],
+    "designationDate": ""
+  },
+  "ContainsSingleDeletion": {
+    "lineage": "B.1.1.515",
+    "unaliased": "B.1.1.515",
+    "parent": "B.1.1",
+    "children": [],
+    "nextstrainClade": "20B",
+    "nucSubstitutions": [
+      "C241T",
+      "C3037T",
+      "C14408T",
+      "G19825T",
+      "A23403G",
+      "G28881A",
+      "G28882A",
+      "G28883C"
+    ],
+    "aaSubstitutions": [
+      "N:R203K",
+      "N:G204R",
+      "ORF1b:P314L",
+      "ORF1b:V2120L",
+      "S:D614G"
+    ],
+    "nucDeletions": [
+      "4"
+    ],
+    "aaDeletions": [
+      ""
+    ],
+    "nucSubstitutionsNew": [
+      "G19825T"
+    ],
+    "aaSubstitutionsNew": [
+      "ORF1b:V2120L"
+    ],
+    "nucDeletionsNew": [
+      "4"
+    ],
+    "aaDeletionsNew": [],
+    "nucSubstitutionsReverted": [],
+    "aaSubstitutionsReverted": [],
+    "nucDeletionsReverted": [
+      ""
+    ],
+    "aaDeletionsReverted": [],
+    "frameShifts": [
+      ""
+    ],
+    "designationDate": "2021-03-15"
+  },
+  "ContainsDeletionRangesAndSingleDeletion": {
+    "lineage": "XAW",
+    "unaliased": "XAW",
+    "parent": "",
+    "children": [],
+    "nextstrainClade": "recombinant",
+    "nucSubstitutions": [
+      "G210T",
+      "C241T",
+      "C335T",
+      "G569A",
+      "T689C",
+      "G1048T",
+      "G1125A",
+      "C1973T",
+      "C3037T",
+      "G3246A",
+      "G4181T",
+      "G4561A",
+      "C5052T",
+      "C5167T",
+      "G6149A",
+      "C6402T",
+      "T6640C",
+      "C7124T",
+      "G7427A",
+      "C8986T",
+      "G9053T",
+      "T9676C",
+      "C10029T",
+      "A10323G",
+      "A11201G",
+      "G11330A",
+      "A11332G",
+      "C11866T",
+      "C13806T",
+      "C13994T",
+      "C14408T",
+      "C14585T",
+      "G15451A",
+      "C16466T",
+      "C17304T",
+      "C17550T",
+      "A17973G",
+      "G18525A",
+      "C19220T",
+      "C19290T",
+      "C20402T",
+      "G20493A",
+      "C21575T",
+      "C21618G",
+      "C21636T",
+      "T22283C",
+      "G22578A",
+      "T22679C",
+      "C22686T",
+      "A22688G",
+      "G22775A",
+      "A22786C",
+      "G22813T",
+      "T22882G",
+      "G22899A",
+      "T22942A",
+      "G22992A",
+      "C22995A",
+      "A23013C",
+      "T23042C",
+      "A23055G",
+      "A23063T",
+      "T23075C",
+      "C23114T",
+      "A23403G",
+      "C23525T",
+      "C23604A",
+      "C23854A",
+      "G23948T",
+      "G24145A",
+      "A24424T",
+      "T24469A",
+      "C25000T",
+      "C25584T",
+      "A25592G",
+      "C26060T",
+      "C26270T",
+      "C26537T",
+      "C26577G",
+      "G26709A",
+      "C26858T",
+      "A27259C",
+      "G27382C",
+      "A27383T",
+      "T27384C",
+      "C27807T",
+      "A28461G",
+      "G28562A",
+      "G28646A",
+      "C28849T",
+      "C28873T",
+      "G28881T",
+      "G28916T",
+      "G29402T",
+      "C29635T",
+      "G29742T"
+    ],
+    "aaSubstitutions": [
+      "E:T9I",
+      "M:Q19E",
+      "M:A63T",
+      "N:D63G",
+      "N:G97S",
+      "N:A125T",
+      "N:R203M",
+      "N:G215C",
+      "N:D377Y",
+      "ORF1a:R24C",
+      "ORF1a:E102K",
+      "ORF1a:S142P",
+      "ORF1a:K261N",
+      "ORF1a:R287K",
+      "ORF1a:S994N",
+      "ORF1a:A1306S",
+      "ORF1a:P1596L",
+      "ORF1a:D1962N",
+      "ORF1a:P2046L",
+      "ORF1a:P2287S",
+      "ORF1a:A2388T",
+      "ORF1a:V2930L",
+      "ORF1a:T3255I",
+      "ORF1a:K3353R",
+      "ORF1a:T3646A",
+      "ORF1a:V3689M",
+      "ORF1b:A176V",
+      "ORF1b:P314L",
+      "ORF1b:A373V",
+      "ORF1b:G662S",
+      "ORF1b:P1000L",
+      "ORF1b:A1918V",
+      "ORF1b:S2312L",
+      "ORF3a:K67R",
+      "ORF3a:T223I",
+      "ORF6:D61L",
+      "ORF9b:T60A",
+      "S:L5F",
+      "S:T19R",
+      "S:P25L",
+      "S:R158G",
+      "S:L212I",
+      "S:G339D",
+      "S:S373P",
+      "S:S375F",
+      "S:T376A",
+      "S:D405N",
+      "S:R408S",
+      "S:K417N",
+      "S:N440K",
+      "S:G446D",
+      "S:N460K",
+      "S:S477N",
+      "S:T478K",
+      "S:E484A",
+      "S:S494P",
+      "S:Q498R",
+      "S:N501Y",
+      "S:Y505H",
+      "S:D614G",
+      "S:H655Y",
+      "S:P681H",
+      "S:N764K",
+      "S:D796Y",
+      "S:Q954H",
+      "S:N969K"
+    ],
+    "nucDeletions": [
+      "22029-22034",
+      "22194-22196",
+      "22289-22294",
+      "28248-28253",
+      "28271"
+    ],
+    "aaDeletions": [
+      "ORF8:D119-",
+      "ORF8:F120-",
+      "S:E156-",
+      "S:F157-",
+      "S:N211-",
+      "S:L242-",
+      "S:A243-"
+    ],
+    "nucSubstitutionsNew": [
+      "G210T",
+      "C241T",
+      "C335T",
+      "G569A",
+      "T689C",
+      "G1048T",
+      "G1125A",
+      "C1973T",
+      "C3037T",
+      "G3246A",
+      "G4181T",
+      "G4561A",
+      "C5052T",
+      "C5167T",
+      "G6149A",
+      "C6402T",
+      "T6640C",
+      "C7124T",
+      "G7427A",
+      "C8986T",
+      "G9053T",
+      "T9676C",
+      "C10029T",
+      "A10323G",
+      "A11201G",
+      "G11330A",
+      "A11332G",
+      "C11866T",
+      "C13806T",
+      "C13994T",
+      "C14408T",
+      "C14585T",
+      "G15451A",
+      "C16466T",
+      "C17304T",
+      "C17550T",
+      "A17973G",
+      "G18525A",
+      "C19220T",
+      "C19290T",
+      "C20402T",
+      "G20493A",
+      "C21575T",
+      "C21618G",
+      "C21636T",
+      "T22283C",
+      "G22578A",
+      "T22679C",
+      "C22686T",
+      "A22688G",
+      "G22775A",
+      "A22786C",
+      "G22813T",
+      "T22882G",
+      "G22899A",
+      "T22942A",
+      "G22992A",
+      "C22995A",
+      "A23013C",
+      "T23042C",
+      "A23055G",
+      "A23063T",
+      "T23075C",
+      "C23114T",
+      "A23403G",
+      "C23525T",
+      "C23604A",
+      "C23854A",
+      "G23948T",
+      "G24145A",
+      "A24424T",
+      "T24469A",
+      "C25000T",
+      "C25584T",
+      "A25592G",
+      "C26060T",
+      "C26270T",
+      "C26537T",
+      "C26577G",
+      "G26709A",
+      "C26858T",
+      "A27259C",
+      "G27382C",
+      "A27383T",
+      "T27384C",
+      "C27807T",
+      "A28461G",
+      "G28562A",
+      "G28646A",
+      "C28849T",
+      "C28873T",
+      "G28881T",
+      "G28916T",
+      "G29402T",
+      "C29635T",
+      "G29742T"
+    ],
+    "aaSubstitutionsNew": [
+      "E:T9I",
+      "M:Q19E",
+      "M:A63T",
+      "N:D63G",
+      "N:G97S",
+      "N:A125T",
+      "N:R203M",
+      "N:G215C",
+      "N:D377Y",
+      "ORF1a:R24C",
+      "ORF1a:E102K",
+      "ORF1a:S142P",
+      "ORF1a:K261N",
+      "ORF1a:R287K",
+      "ORF1a:S994N",
+      "ORF1a:A1306S",
+      "ORF1a:P1596L",
+      "ORF1a:D1962N",
+      "ORF1a:P2046L",
+      "ORF1a:P2287S",
+      "ORF1a:A2388T",
+      "ORF1a:V2930L",
+      "ORF1a:T3255I",
+      "ORF1a:K3353R",
+      "ORF1a:T3646A",
+      "ORF1a:V3689M",
+      "ORF1b:A176V",
+      "ORF1b:P314L",
+      "ORF1b:A373V",
+      "ORF1b:G662S",
+      "ORF1b:P1000L",
+      "ORF1b:A1918V",
+      "ORF1b:S2312L",
+      "ORF3a:K67R",
+      "ORF3a:T223I",
+      "ORF6:D61L",
+      "ORF9b:T60A",
+      "S:L5F",
+      "S:T19R",
+      "S:P25L",
+      "S:R158G",
+      "S:L212I",
+      "S:G339D",
+      "S:S373P",
+      "S:S375F",
+      "S:T376A",
+      "S:D405N",
+      "S:R408S",
+      "S:K417N",
+      "S:N440K",
+      "S:G446D",
+      "S:N460K",
+      "S:S477N",
+      "S:T478K",
+      "S:E484A",
+      "S:S494P",
+      "S:Q498R",
+      "S:N501Y",
+      "S:Y505H",
+      "S:D614G",
+      "S:H655Y",
+      "S:P681H",
+      "S:N764K",
+      "S:D796Y",
+      "S:Q954H",
+      "S:N969K"
+    ],
+    "nucDeletionsNew": [
+      "22029-22034",
+      "22194-22196",
+      "22289-22294",
+      "28248-28253",
+      "28271"
+    ],
+    "aaDeletionsNew": [
+      "ORF8:D119-",
+      "ORF8:F120-",
+      "S:E156-",
+      "S:F157-",
+      "S:N211-",
+      "S:L242-",
+      "S:A243-"
+    ],
+    "nucSubstitutionsReverted": [],
+    "aaSubstitutionsReverted": [],
+    "nucDeletionsReverted": [],
+    "aaDeletionsReverted": [],
+    "frameShifts": [
+      ""
+    ],
+    "designationDate": "2022-08-25"
+  }
+}
diff --git a/tests/data/defining_mutations/expected_output/BE.1.1.1.json b/tests/data/defining_mutations/expected_output/BE.1.1.1.json
new file mode 100644
index 0000000000..dd8aeef52e
--- /dev/null
+++ b/tests/data/defining_mutations/expected_output/BE.1.1.1.json
@@ -0,0 +1 @@
+{"lineage": "BE.1.1.1", "nextstrain_clade": "22B", "mutations": [{"reference": "pango_parent", "coding": [{"aa_mutation": {"gene": "ORF9b", "ref": "P", "pos": 10, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 28312, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "H", "pos": 69, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 21765, "alt": "-"}, {"ref": "A", "pos": 21766, "alt": "-"}, {"ref": "C", "pos": 21767, "alt": "-"}]}, {"aa_mutation": {"gene": "S", "ref": "L", "pos": 452, "alt": "R"}, "nuc_mutations": [{"ref": "T", "pos": 22917, "alt": "G"}]}, {"aa_mutation": {"gene": "M", "ref": "D", "pos": 3, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 26529, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "V", "pos": 70, "alt": "-"}, "nuc_mutations": [{"ref": "A", "pos": 21768, "alt": "-"}, {"ref": "T", "pos": 21769, "alt": "-"}, {"ref": "G", "pos": 21770, "alt": "-"}]}, {"aa_mutation": {"gene": "S", "ref": "F", "pos": 486, "alt": "V"}, "nuc_mutations": [{"ref": "T", "pos": 23018, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "K", "pos": 444, "alt": "T"}, "nuc_mutations": [{"ref": "A", "pos": 22893, "alt": "C"}]}], "silent": [{"nuc_mutation": {"ref": "C", "pos": 27889, "alt": "T"}}]}, {"reference": "wuhan", "coding": [{"aa_mutation": {"gene": "ORF1a", "ref": "T", "pos": 842, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 2790, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "D", "pos": 405, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 22775, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "E", "pos": 31, "alt": "-"}, "nuc_mutations": [{"ref": "G", "pos": 28362, "alt": "-"}, {"ref": "A", "pos": 28363, "alt": "-"}, {"ref": "G", "pos": 28364, "alt": "-"}], "notes": "Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33"}, {"aa_mutation": {"gene": "S", "ref": "A", "pos": 27, "alt": "S"}, "nuc_mutations": [{"ref": "G", "pos": 21641, "alt": "-"}], "notes": "This change is a consequence of the prior deletion"}, {"aa_mutation": {"gene": "ORF1a", "ref": "S", "pos": 3675, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 11288, "alt": "-"}, {"ref": "C", "pos": 11289, "alt": "-"}, {"ref": "T", "pos": 11290, "alt": "-"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "P", "pos": 3395, "alt": "H"}, "nuc_mutations": [{"ref": "C", "pos": 10449, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 969, "alt": "K"}, "nuc_mutations": [{"ref": "T", "pos": 24469, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "S", "pos": 371, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 22674, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "V", "pos": 213, "alt": "G"}, "nuc_mutations": [{"ref": "T", "pos": 22200, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 440, "alt": "K"}, "nuc_mutations": [{"ref": "T", "pos": 22882, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "D", "pos": 614, "alt": "G"}, "nuc_mutations": [{"ref": "A", "pos": 23403, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "G", "pos": 339, "alt": "D"}, "nuc_mutations": [{"ref": "G", "pos": 22578, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "R", "pos": 32, "alt": "-"}, "nuc_mutations": [{"ref": "A", "pos": 28365, "alt": "-"}, {"ref": "A", "pos": 28366, "alt": "-"}, {"ref": "C", "pos": 28367, "alt": "-"}], "notes": "Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33"}, {"aa_mutation": {"gene": "N", "ref": "S", "pos": 413, "alt": "R"}, "nuc_mutations": [{"ref": "A", "pos": 29510, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 679, "alt": "K"}, "nuc_mutations": [{"ref": "T", "pos": 23599, "alt": "G"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "G", "pos": 1307, "alt": "S"}, "nuc_mutations": [{"ref": "G", "pos": 4184, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "S", "pos": 477, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 22992, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "Q", "pos": 954, "alt": "H"}, "nuc_mutations": [{"ref": "A", "pos": 24424, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 764, "alt": "K"}, "nuc_mutations": [{"ref": "C", "pos": 23854, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "R", "pos": 1315, "alt": "C"}, "nuc_mutations": [{"ref": "C", "pos": 17410, "alt": "T"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "P", "pos": 314, "alt": "L"}, "nuc_mutations": [{"ref": "C", "pos": 14408, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "L", "pos": 452, "alt": "R"}, "nuc_mutations": [{"ref": "T", "pos": 22917, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "T", "pos": 376, "alt": "A"}, "nuc_mutations": [{"ref": "A", "pos": 22688, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "Q", "pos": 498, "alt": "R"}, "nuc_mutations": [{"ref": "A", "pos": 23055, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "P", "pos": 25, "alt": "-"}, "nuc_mutations": [{"ref": "C", "pos": 21635, "alt": "-"}, {"ref": "C", "pos": 21636, "alt": "-"}, {"ref": "C", "pos": 21637, "alt": "-"}], "notes": "Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27"}, {"aa_mutation": {"gene": "M", "ref": "D", "pos": 3, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 26529, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "R", "pos": 203, "alt": "K"}, "nuc_mutations": [{"ref": "G", "pos": 28881, "alt": "A"}, {"ref": "G", "pos": 28882, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "S", "pos": 135, "alt": "R"}, "nuc_mutations": [{"ref": "T", "pos": 670, "alt": "G"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "I", "pos": 1566, "alt": "V"}, "nuc_mutations": [{"ref": "A", "pos": 18163, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "H", "pos": 69, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 21765, "alt": "-"}, {"ref": "A", "pos": 21766, "alt": "-"}, {"ref": "C", "pos": 21767, "alt": "-"}]}, {"aa_mutation": {"gene": "N", "ref": "P", "pos": 13, "alt": "L"}, "nuc_mutations": [{"ref": "C", "pos": 28311, "alt": "T"}, {"ref": "C", "pos": 28312, "alt": "T"}]}, {"aa_mutation": {"gene": "E", "ref": "T", "pos": 9, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 26270, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "P", "pos": 26, "alt": "-"}, "nuc_mutations": [{"ref": "C", "pos": 21638, "alt": "-"}, {"ref": "C", "pos": 21639, "alt": "-"}, {"ref": "T", "pos": 21640, "alt": "-"}], "notes": "Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27"}, {"aa_mutation": {"gene": "S", "ref": "E", "pos": 484, "alt": "A"}, "nuc_mutations": [{"ref": "A", "pos": 23013, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "R", "pos": 408, "alt": "S"}, "nuc_mutations": [{"ref": "A", "pos": 22786, "alt": "C"}]}, {"aa_mutation": {"gene": "M", "ref": "A", "pos": 63, "alt": "T"}, "nuc_mutations": [{"ref": "G", "pos": 26709, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "H", "pos": 655, "alt": "Y"}, "nuc_mutations": [{"ref": "C", "pos": 23525, "alt": "T"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "Q", "pos": 556, "alt": "K"}, "nuc_mutations": [{"ref": "C", "pos": 1931, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "G", "pos": 204, "alt": "R"}, "nuc_mutations": [{"ref": "G", "pos": 28883, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "S", "pos": 373, "alt": "P"}, "nuc_mutations": [{"ref": "T", "pos": 22679, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "G", "pos": 142, "alt": "D"}, "nuc_mutations": [{"ref": "G", "pos": 21987, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "G", "pos": 3676, "alt": "-"}, "nuc_mutations": [{"ref": "G", "pos": 11291, "alt": "-"}, {"ref": "G", "pos": 11292, "alt": "-"}, {"ref": "T", "pos": 11293, "alt": "-"}]}, {"aa_mutation": {"gene": "ORF3a", "ref": "T", "pos": 223, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 26060, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "F", "pos": 486, "alt": "V"}, "nuc_mutations": [{"ref": "T", "pos": 23018, "alt": "G"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "T", "pos": 3255, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 10029, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "S", "pos": 375, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 22686, "alt": "T"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "F", "pos": 3677, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 11294, "alt": "-"}, {"ref": "T", "pos": 11295, "alt": "-"}, {"ref": "T", "pos": 11296, "alt": "-"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 501, "alt": "Y"}, "nuc_mutations": [{"ref": "A", "pos": 23063, "alt": "T"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "L", "pos": 3829, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 11750, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "K", "pos": 417, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 22813, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "L", "pos": 24, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 21633, "alt": "-"}, {"ref": "A", "pos": 21634, "alt": "-"}], "notes": "Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27"}, {"aa_mutation": {"gene": "ORF1b", "ref": "M", "pos": 1156, "alt": "I"}, "nuc_mutations": [{"ref": "G", "pos": 16935, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "T", "pos": 2163, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 19955, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "P", "pos": 681, "alt": "H"}, "nuc_mutations": [{"ref": "C", "pos": 23604, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "K", "pos": 444, "alt": "T"}, "nuc_mutations": [{"ref": "A", "pos": 22893, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "T", "pos": 478, "alt": "K"}, "nuc_mutations": [{"ref": "C", "pos": 22995, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "E", "pos": 136, "alt": "D"}, "nuc_mutations": [{"ref": "G", "pos": 28681, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "V", "pos": 70, "alt": "-"}, "nuc_mutations": [{"ref": "A", "pos": 21768, "alt": "-"}, {"ref": "T", "pos": 21769, "alt": "-"}, {"ref": "G", "pos": 21770, "alt": "-"}]}, {"aa_mutation": {"gene": "M", "ref": "Q", "pos": 19, "alt": "E"}, "nuc_mutations": [{"ref": "C", "pos": 26577, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "D", "pos": 796, "alt": "Y"}, "nuc_mutations": [{"ref": "G", "pos": 23948, "alt": "T"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "L", "pos": 3027, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 9344, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "Y", "pos": 505, "alt": "H"}, "nuc_mutations": [{"ref": "T", "pos": 23075, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "T", "pos": 19, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 21618, "alt": "T"}]}, {"aa_mutation": {"gene": "N", "ref": "S", "pos": 33, "alt": "-"}, "nuc_mutations": [{"ref": "G", "pos": 28368, "alt": "-"}, {"ref": "C", "pos": 28369, "alt": "-"}, {"ref": "A", "pos": 28370, "alt": "-"}], "notes": "Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33"}, {"aa_mutation": {"gene": "ORF1a", "ref": "T", "pos": 3090, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 9534, "alt": "T"}]}], "silent": [{"nuc_mutation": {"ref": "C", "pos": 241, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 3037, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 4321, "alt": "T"}}, {"nuc_mutation": {"ref": "A", "pos": 9424, "alt": "G"}}, {"nuc_mutation": {"ref": "C", "pos": 10198, "alt": "T"}}, {"nuc_mutation": {"ref": "G", "pos": 10447, "alt": "A"}}, {"nuc_mutation": {"ref": "G", "pos": 12160, "alt": "A"}}, {"nuc_mutation": {"ref": "C", "pos": 12880, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 15714, "alt": "T"}}, {"nuc_mutation": {"ref": "A", "pos": 20055, "alt": "G"}}, {"nuc_mutation": {"ref": "C", "pos": 25000, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 25584, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 27807, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 27889, "alt": "T"}}, {"nuc_mutation": {"ref": "A", "pos": 28271, "alt": "T"}}, {"nuc_mutation": {"ref": "G", "pos": 29734, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29735, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29736, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29737, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29738, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29739, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29740, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29741, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29742, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29743, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29744, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29745, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29746, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29747, "alt": "-"}}, {"nuc_mutation": {"ref": "T", "pos": 29748, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29749, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29750, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29751, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29752, "alt": "-"}}, {"nuc_mutation": {"ref": "T", "pos": 29753, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29754, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29755, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29756, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29757, "alt": "-"}}, {"nuc_mutation": {"ref": "T", "pos": 29758, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29759, "alt": "-"}}, {"nuc_mutation": {"ref": "T", "pos": 2954, "alt": "C"}}, {"nuc_mutation": {"ref": "G", "pos": 29868, "alt": "A"}}]}], "children": ["BQ.1", "BQ.2"], "designation_date": "2022-08-24", "unaliased": "B.1.1.529.5.3.1.1.1.1", "parent": "BE.1.1"}
\ No newline at end of file
diff --git a/tests/data/defining_mutations/expected_output/BQ.1.json b/tests/data/defining_mutations/expected_output/BQ.1.json
new file mode 100644
index 0000000000..e3224b0a5a
--- /dev/null
+++ b/tests/data/defining_mutations/expected_output/BQ.1.json
@@ -0,0 +1 @@
+{"lineage": "BQ.1", "nextstrain_clade": "22E", "mutations": [{"reference": "pango_parent", "coding": [{"aa_mutation": {"gene": "S", "ref": "N", "pos": 460, "alt": "K"}, "nuc_mutations": [{"ref": "T", "pos": 22942, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "Y", "pos": 264, "alt": "H"}, "nuc_mutations": [{"ref": "T", "pos": 14257, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "K", "pos": 444, "alt": "T"}, "nuc_mutations": [{"ref": "A", "pos": 22893, "alt": "C"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "Q", "pos": 556, "alt": "K"}, "nuc_mutations": [{"ref": "C", "pos": 1931, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "P", "pos": 13, "alt": "L"}, "nuc_mutations": [{"ref": "C", "pos": 28312, "alt": "T"}], "notes": "This mutation on top of the one above, change ORf9b:P10S to P10F"}, {"aa_mutation": {"gene": "ORF1a", "ref": "L", "pos": 3829, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 11750, "alt": "T"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "M", "pos": 1156, "alt": "I"}, "nuc_mutations": [{"ref": "G", "pos": 16935, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "E", "pos": 136, "alt": "D"}, "nuc_mutations": [{"ref": "G", "pos": 28681, "alt": "T"}]}], "silent": [{"nuc_mutation": {"ref": "T", "pos": 2954, "alt": "C"}}]}, {"reference": "wuhan", "coding": [{"aa_mutation": {"gene": "S", "ref": "N", "pos": 969, "alt": "K"}, "nuc_mutations": [{"ref": "T", "pos": 24469, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "S", "pos": 477, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 22992, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 679, "alt": "K"}, "nuc_mutations": [{"ref": "T", "pos": 23599, "alt": "G"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "P", "pos": 314, "alt": "L"}, "nuc_mutations": [{"ref": "C", "pos": 14408, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "P", "pos": 26, "alt": "-"}, "nuc_mutations": [{"ref": "C", "pos": 21638, "alt": "-"}, {"ref": "C", "pos": 21639, "alt": "-"}, {"ref": "T", "pos": 21640, "alt": "-"}], "notes": "Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27"}, {"aa_mutation": {"gene": "N", "ref": "E", "pos": 136, "alt": "D"}, "nuc_mutations": [{"ref": "G", "pos": 28681, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "G", "pos": 142, "alt": "D"}, "nuc_mutations": [{"ref": "G", "pos": 21987, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "E", "pos": 31, "alt": "-"}, "nuc_mutations": [{"ref": "G", "pos": 28362, "alt": "-"}, {"ref": "A", "pos": 28363, "alt": "-"}, {"ref": "G", "pos": 28364, "alt": "-"}], "notes": "Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33"}, {"aa_mutation": {"gene": "S", "ref": "S", "pos": 375, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 22686, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "Q", "pos": 954, "alt": "H"}, "nuc_mutations": [{"ref": "A", "pos": 24424, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "H", "pos": 655, "alt": "Y"}, "nuc_mutations": [{"ref": "C", "pos": 23525, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "L", "pos": 452, "alt": "R"}, "nuc_mutations": [{"ref": "T", "pos": 22917, "alt": "G"}]}, {"aa_mutation": {"gene": "N", "ref": "S", "pos": 33, "alt": "-"}, "nuc_mutations": [{"ref": "G", "pos": 28368, "alt": "-"}, {"ref": "C", "pos": 28369, "alt": "-"}, {"ref": "A", "pos": 28370, "alt": "-"}], "notes": "Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33"}, {"aa_mutation": {"gene": "S", "ref": "G", "pos": 339, "alt": "D"}, "nuc_mutations": [{"ref": "G", "pos": 22578, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "Q", "pos": 556, "alt": "K"}, "nuc_mutations": [{"ref": "C", "pos": 1931, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "M", "pos": 1156, "alt": "I"}, "nuc_mutations": [{"ref": "G", "pos": 16935, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "P", "pos": 3395, "alt": "H"}, "nuc_mutations": [{"ref": "C", "pos": 10449, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "E", "pos": 484, "alt": "A"}, "nuc_mutations": [{"ref": "A", "pos": 23013, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 440, "alt": "K"}, "nuc_mutations": [{"ref": "T", "pos": 22882, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "T", "pos": 19, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 21618, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "T", "pos": 376, "alt": "A"}, "nuc_mutations": [{"ref": "A", "pos": 22688, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "D", "pos": 796, "alt": "Y"}, "nuc_mutations": [{"ref": "G", "pos": 23948, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "Q", "pos": 498, "alt": "R"}, "nuc_mutations": [{"ref": "A", "pos": 23055, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "P", "pos": 25, "alt": "-"}, "nuc_mutations": [{"ref": "C", "pos": 21635, "alt": "-"}, {"ref": "C", "pos": 21636, "alt": "-"}, {"ref": "C", "pos": 21637, "alt": "-"}], "notes": "Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27"}, {"aa_mutation": {"gene": "E", "ref": "T", "pos": 9, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 26270, "alt": "T"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "Y", "pos": 264, "alt": "H"}, "nuc_mutations": [{"ref": "T", "pos": 14257, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 460, "alt": "K"}, "nuc_mutations": [{"ref": "T", "pos": 22942, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 764, "alt": "K"}, "nuc_mutations": [{"ref": "C", "pos": 23854, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "R", "pos": 1315, "alt": "C"}, "nuc_mutations": [{"ref": "C", "pos": 17410, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "A", "pos": 27, "alt": "S"}, "nuc_mutations": [{"ref": "G", "pos": 21641, "alt": "-"}], "notes": "This change is a consequence of the prior deletion"}, {"aa_mutation": {"gene": "S", "ref": "N", "pos": 501, "alt": "Y"}, "nuc_mutations": [{"ref": "A", "pos": 23063, "alt": "T"}]}, {"aa_mutation": {"gene": "M", "ref": "D", "pos": 3, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 26529, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "S", "pos": 371, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 22674, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "S", "pos": 373, "alt": "P"}, "nuc_mutations": [{"ref": "T", "pos": 22679, "alt": "C"}]}, {"aa_mutation": {"gene": "S", "ref": "T", "pos": 478, "alt": "K"}, "nuc_mutations": [{"ref": "C", "pos": 22995, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "T", "pos": 2163, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 19955, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "V", "pos": 70, "alt": "-"}, "nuc_mutations": [{"ref": "A", "pos": 21768, "alt": "-"}, {"ref": "T", "pos": 21769, "alt": "-"}, {"ref": "G", "pos": 21770, "alt": "-"}]}, {"aa_mutation": {"gene": "S", "ref": "V", "pos": 213, "alt": "G"}, "nuc_mutations": [{"ref": "T", "pos": 22200, "alt": "G"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "G", "pos": 3676, "alt": "-"}, "nuc_mutations": [{"ref": "G", "pos": 11291, "alt": "-"}, {"ref": "G", "pos": 11292, "alt": "-"}, {"ref": "T", "pos": 11293, "alt": "-"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "T", "pos": 842, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 2790, "alt": "T"}]}, {"aa_mutation": {"gene": "ORF3a", "ref": "T", "pos": 223, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 26060, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "L", "pos": 24, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 21633, "alt": "-"}, {"ref": "A", "pos": 21634, "alt": "-"}], "notes": "Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27"}, {"aa_mutation": {"gene": "ORF1a", "ref": "L", "pos": 3829, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 11750, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "P", "pos": 681, "alt": "H"}, "nuc_mutations": [{"ref": "C", "pos": 23604, "alt": "A"}]}, {"aa_mutation": {"gene": "S", "ref": "K", "pos": 417, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 22813, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "R", "pos": 408, "alt": "S"}, "nuc_mutations": [{"ref": "A", "pos": 22786, "alt": "C"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "L", "pos": 3027, "alt": "F"}, "nuc_mutations": [{"ref": "C", "pos": 9344, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "D", "pos": 614, "alt": "G"}, "nuc_mutations": [{"ref": "A", "pos": 23403, "alt": "G"}]}, {"aa_mutation": {"gene": "ORF1b", "ref": "I", "pos": 1566, "alt": "V"}, "nuc_mutations": [{"ref": "A", "pos": 18163, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "D", "pos": 405, "alt": "N"}, "nuc_mutations": [{"ref": "G", "pos": 22775, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "S", "pos": 135, "alt": "R"}, "nuc_mutations": [{"ref": "T", "pos": 670, "alt": "G"}]}, {"aa_mutation": {"gene": "S", "ref": "F", "pos": 486, "alt": "V"}, "nuc_mutations": [{"ref": "T", "pos": 23018, "alt": "G"}]}, {"aa_mutation": {"gene": "M", "ref": "A", "pos": 63, "alt": "T"}, "nuc_mutations": [{"ref": "G", "pos": 26709, "alt": "A"}]}, {"aa_mutation": {"gene": "N", "ref": "R", "pos": 32, "alt": "-"}, "nuc_mutations": [{"ref": "A", "pos": 28365, "alt": "-"}, {"ref": "A", "pos": 28366, "alt": "-"}, {"ref": "C", "pos": 28367, "alt": "-"}], "notes": "Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33"}, {"aa_mutation": {"gene": "ORF1a", "ref": "T", "pos": 3090, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 9534, "alt": "T"}]}, {"aa_mutation": {"gene": "S", "ref": "H", "pos": 69, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 21765, "alt": "-"}, {"ref": "A", "pos": 21766, "alt": "-"}, {"ref": "C", "pos": 21767, "alt": "-"}]}, {"aa_mutation": {"gene": "N", "ref": "G", "pos": 204, "alt": "R"}, "nuc_mutations": [{"ref": "G", "pos": 28883, "alt": "C"}]}, {"aa_mutation": {"gene": "N", "ref": "S", "pos": 413, "alt": "R"}, "nuc_mutations": [{"ref": "A", "pos": 29510, "alt": "C"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "T", "pos": 3255, "alt": "I"}, "nuc_mutations": [{"ref": "C", "pos": 10029, "alt": "T"}]}, {"aa_mutation": {"gene": "N", "ref": "P", "pos": 13, "alt": "L"}, "nuc_mutations": [{"ref": "C", "pos": 28311, "alt": "T"}, {"ref": "C", "pos": 28312, "alt": "T"}], "notes": "This mutation causes ORG9b:P10S, but the adjacent mutation changes to P10F"}, {"aa_mutation": {"gene": "S", "ref": "Y", "pos": 505, "alt": "H"}, "nuc_mutations": [{"ref": "T", "pos": 23075, "alt": "C"}]}, {"aa_mutation": {"gene": "N", "ref": "R", "pos": 203, "alt": "K"}, "nuc_mutations": [{"ref": "G", "pos": 28881, "alt": "A"}, {"ref": "G", "pos": 28882, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "G", "pos": 1307, "alt": "S"}, "nuc_mutations": [{"ref": "G", "pos": 4184, "alt": "A"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "S", "pos": 3675, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 11288, "alt": "-"}, {"ref": "C", "pos": 11289, "alt": "-"}, {"ref": "T", "pos": 11290, "alt": "-"}]}, {"aa_mutation": {"gene": "ORF1a", "ref": "F", "pos": 3677, "alt": "-"}, "nuc_mutations": [{"ref": "T", "pos": 11294, "alt": "-"}, {"ref": "T", "pos": 11295, "alt": "-"}, {"ref": "T", "pos": 11296, "alt": "-"}]}, {"aa_mutation": {"gene": "S", "ref": "K", "pos": 444, "alt": "T"}, "nuc_mutations": [{"ref": "A", "pos": 22893, "alt": "C"}]}, {"aa_mutation": {"gene": "M", "ref": "Q", "pos": 19, "alt": "E"}, "nuc_mutations": [{"ref": "C", "pos": 26577, "alt": "G"}]}], "silent": [{"nuc_mutation": {"ref": "C", "pos": 241, "alt": "T"}}, {"nuc_mutation": {"ref": "T", "pos": 2954, "alt": "C"}}, {"nuc_mutation": {"ref": "C", "pos": 3037, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 4321, "alt": "T"}}, {"nuc_mutation": {"ref": "A", "pos": 9424, "alt": "G"}}, {"nuc_mutation": {"ref": "C", "pos": 10198, "alt": "T"}}, {"nuc_mutation": {"ref": "G", "pos": 10447, "alt": "A"}}, {"nuc_mutation": {"ref": "G", "pos": 12160, "alt": "A"}}, {"nuc_mutation": {"ref": "C", "pos": 12880, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 15714, "alt": "T"}}, {"nuc_mutation": {"ref": "A", "pos": 20055, "alt": "G"}}, {"nuc_mutation": {"ref": "C", "pos": 25000, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 25584, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 27807, "alt": "T"}}, {"nuc_mutation": {"ref": "C", "pos": 27889, "alt": "T"}}, {"nuc_mutation": {"ref": "A", "pos": 28271, "alt": "T"}}, {"nuc_mutation": {"ref": "G", "pos": 29734, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29735, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29736, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29737, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29738, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29739, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29740, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29741, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29742, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29743, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29744, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29745, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29746, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29747, "alt": "-"}}, {"nuc_mutation": {"ref": "T", "pos": 29748, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29749, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29750, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29751, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29752, "alt": "-"}}, {"nuc_mutation": {"ref": "T", "pos": 29753, "alt": "-"}}, {"nuc_mutation": {"ref": "C", "pos": 29754, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29755, "alt": "-"}}, {"nuc_mutation": {"ref": "A", "pos": 29756, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29757, "alt": "-"}}, {"nuc_mutation": {"ref": "T", "pos": 29758, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29759, "alt": "-"}}, {"nuc_mutation": {"ref": "G", "pos": 29868, "alt": "A"}}]}], "children": ["BQ.1.1", "BQ.1.2", "BQ.1.3", "BQ.1.4", "BQ.1.5", "BQ.1.6", "BQ.1.7", "BQ.1.8", "BQ.1.9", "BQ.1.10", "BQ.1.11", "BQ.1.12", "BQ.1.13", "BQ.1.14", "BQ.1.15", "BQ.1.16", "BQ.1.17", "BQ.1.18", "BQ.1.19", "BQ.1.20", "BQ.1.21", "BQ.1.22", "BQ.1.23", "BQ.1.24", "BQ.1.25", "BQ.1.26", "BQ.1.27", "BQ.1.28", "BQ.1.29", "BQ.1.30", "BQ.1.31", "BQ.1.32", "BQ.1.33"], "designation_date": "2022-09-02", "unaliased": "B.1.1.529.5.3.1.1.1.1.1", "parent": "BE.1.1.1"}
\ No newline at end of file
diff --git a/tests/data/defining_mutations/expected_output/definingMutationsClusters.json b/tests/data/defining_mutations/expected_output/definingMutationsClusters.json
new file mode 100644
index 0000000000..c1d4eb091e
--- /dev/null
+++ b/tests/data/defining_mutations/expected_output/definingMutationsClusters.json
@@ -0,0 +1,12 @@
+{
+  "clusters": [
+    {
+      "lineage": "BQ.1",
+      "nextstrain_clade": "22E"
+    },
+    {
+      "lineage": "BE.1.1.1",
+      "nextstrain_clade": "22B"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/tests/data/defining_mutations/hand_curated/22B.Omicron.tsv b/tests/data/defining_mutations/hand_curated/22B.Omicron.tsv
new file mode 100644
index 0000000000..856d8d040f
--- /dev/null
+++ b/tests/data/defining_mutations/hand_curated/22B.Omicron.tsv
@@ -0,0 +1,131 @@
+nuc_change	aa_change	aa_change_2	not_in_parent	reversion	notes
+C241T					
+T670G	ORF1a:S135R				
+C2790T	ORF1a:T842I				
+C3037T					
+G4184A	ORF1a:G1307S				
+C4321T					
+C9344T	ORF1a:L3027F				
+A9424G					
+C9534T	ORF1a:T3090I				
+C10029T	ORF1a:T3255I				
+C10198T					
+G10447A					
+C10449A	ORF1a:P3395H				
+T11288-	ORF1a:S3675-				
+C11289-	ORF1a:S3675-				
+T11290-	ORF1a:S3675-				
+G11291-	ORF1a:G3676-				
+G11292-	ORF1a:G3676-				
+T11293-	ORF1a:G3676-				
+T11294-	ORF1a:F3677-				
+T11295-	ORF1a:F3677-				
+T11296-	ORF1a:F3677-				
+G12160A					
+C12880T					
+C14408T	ORF1b:P314L				
+C15714T					
+C17410T	ORF1b:R1315C				
+A18163G	ORF1b:I1566V				
+C19955T	ORF1b:T2163I				
+A20055G					
+C21618T	S:T19I				
+T21633-	S:L24-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+A21634-	S:L24-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21635-	S:P25-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21636-	S:P25-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21637-	S:P25-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21638-	S:P26-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21639-	S:P26-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+T21640-	S:P26-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+G21641-	S:A27S				This change is a consequence of the prior deletion
+T21765-	S:H69-		y		
+A21766-	S:H69-		y		
+C21767-	S:H69-		y		
+A21768-	S:V70-		y		
+T21769-	S:V70-		y		
+G21770-	S:V70-		y		
+G21987A	S:G142D				
+T22200G	S:V213G				
+G22578A	S:G339D				
+C22674T	S:S371F				
+T22679C	S:S373P				
+C22686T	S:S375F				
+A22688G	S:T376A				
+G22775A	S:D405N				
+A22786C	S:R408S				
+G22813T	S:K417N				
+T22882G	S:N440K				
+T22917G	S:L452R		y		
+G22992A	S:S477N				
+C22995A	S:T478K				
+A23013C	S:E484A				
+T23018G	S:F486V		y		
+A23040A	S:Q493Q		y	y	Note mutation S:Q493R (in 21L) is reverted here
+A23055G	S:Q498R				
+A23063T	S:N501Y				
+T23075C	S:Y505H				
+A23403G	S:D614G				
+C23525T	S:H655Y				
+T23599G	S:N679K				
+C23604A	S:P681H				
+C23854A	S:N764K				
+G23948T	S:D796Y				
+A24424T	S:Q954H				
+T24469A	S:N969K				
+C25000T					
+C25584T					
+C26060T	ORF3a:T223I				
+C26270T	E:T9I				
+G26529A	M:D3N		y		
+C26577G	M:Q19E				
+G26709A	M:A63T				
+C26858C			y	y	Reverted relative to 21L
+G27382G	ORF6:D61D		y	y	Note mutation in ORF6:D61L (in 21L) is reverted here
+A27259A			y	y	Reverted realtive to 21L
+G27382G			y	y	Reverted relative to 21L
+A27383A			y	y	Reverted relative to 21L
+T27384T			y	y	Reverted relative to 21L
+C27807T					
+C27889T			y		
+A28271T					
+C28311T	N:P13L	ORF9b:P10S			
+G28362-	N:E31-	ORF9b:E27-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+A28363-	N:E31-	ORF9b:E27-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+G28364-	N:E31-	ORF9b:E27-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+A28365-	N:R32-	ORF9b:N28-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+A28366-	N:R32-	ORF9b:N28-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+C28367-	N:R32-	ORF9b:N28-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+G28368-	N:S33-	ORF9b:A29-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+C28369-	N:S33-	ORF9b:A29-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+A28370-	N:S33-	ORF9b:A29-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+G28881A	N:R203K				
+G28882A	N:R203K				
+G28883C	N:G204R				
+A29510C	N:S413R				
+G29734-					
+A29735-					
+G29736-					
+G29737-					
+C29738-					
+C29739-					
+A29740-					
+C29741-					
+G29742-					
+C29743-					
+G29744-					
+G29745-					
+A29746-					
+G29747-					
+T29748-					
+A29749-					
+C29750-					
+G29751-					
+A29752-					
+T29753-					
+C29754-					
+G29755-					
+A29756-					
+G29757-					
+T29758-					
+G29759-					
\ No newline at end of file
diff --git a/tests/data/defining_mutations/hand_curated/22E.Omicron.tsv b/tests/data/defining_mutations/hand_curated/22E.Omicron.tsv
new file mode 100644
index 0000000000..5227065009
--- /dev/null
+++ b/tests/data/defining_mutations/hand_curated/22E.Omicron.tsv
@@ -0,0 +1,140 @@
+nuc_change	aa_change	aa_change_2	not_in_parent	reversion	notes
+C241T					
+T670G	ORF1a:S135R				
+C1931A	ORF1a:Q556K		y		
+C2790T	ORF1a:T842I				
+T2954C			y		
+C3037T					
+G4184A	ORF1a:G1307S				
+C4321T					
+C9344T	ORF1a:L3027F				
+A9424G					
+C9534T	ORF1a:T3090I				
+C10029T	ORF1a:T3255I				
+C10198T					
+G10447A					
+C10449A	ORF1a:P3395H				
+T11288-	ORF1a:S3675-				
+C11289-	ORF1a:S3675-				
+T11290-	ORF1a:S3675-				
+G11291-	ORF1a:G3676-				
+G11292-	ORF1a:G3676-				
+T11293-	ORF1a:G3676-				
+T11294-	ORF1a:F3677-				
+T11295-	ORF1a:F3677-				
+T11296-	ORF1a:F3677-				
+C11750T	ORF1a:L3829F		y		
+G12160A					
+C12880T					
+T14257C	ORF1b:Y264H		y		
+C14408T	ORF1b:P314L				
+C15714T					
+G16935A	ORF1b:M1156I		y		
+C17410T	ORF1b:R1315C				
+A18163G	ORF1b:I1566V				
+C19955T	ORF1b:T2163I				
+A20055G					
+C21618T	S:T19I				
+T21633-	S:L24-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+A21634-	S:L24-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21635-	S:P25-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21636-	S:P25-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21637-	S:P25-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21638-	S:P26-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+C21639-	S:P26-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+T21640-	S:P26-				Deletion removes last 2 bases from AA 24, entirety of 25/26, first base of 27
+G21641-	S:A27S				This change is a consequence of the prior deletion
+T21765-	S:H69-				
+A21766-	S:H69-				
+C21767-	S:H69-				
+A21768-	S:V70-				
+T21769-	S:V70-				
+G21770-	S:V70-				
+G21987A	S:G142D				
+T22200G	S:V213G				
+G22578A	S:G339D				
+C22674T	S:S371F				
+T22679C	S:S373P				
+C22686T	S:S375F				
+A22688G	S:T376A				
+G22775A	S:D405N				
+A22786C	S:R408S				
+G22813T	S:K417N				
+T22882G	S:N440K				
+A22893C	S:K444T		y		
+T22917G	S:L452R				
+T22942A	S:N460K		y		
+G22992A	S:S477N				
+C22995A	S:T478K				
+A23013C	S:E484A				
+T23018G	S:F486V				
+A23040A	S:Q493Q			y	Note mutation S:Q493R (in 21L) is reverted here
+A23055G	S:Q498R				
+A23063T	S:N501Y				
+T23075C	S:Y505H				
+A23403G	S:D614G				
+C23525T	S:H655Y				
+T23599G	S:N679K				
+C23604A	S:P681H				
+C23854A	S:N764K				
+G23948T	S:D796Y				
+A24424T	S:Q954H				
+T24469A	S:N969K				
+C25000T					
+C25584T					
+C26060T	ORF3a:T223I				
+C26270T	E:T9I				
+G26529A	M:D3N				
+C26577G	M:Q19E				
+G26709A	M:A63T				
+C26858C				y	Reverted relative to 21L
+G27382G	ORF6:D61D			y	Note mutation in ORF6:D61L (in 21L) is reverted here
+A27259A				y	Reverted relative to 21L
+G27382G				y	Reverted relative to 21L
+A27383A				y	Reverted relative to 21L
+T27384T				y	Reverted relative to 21L
+C27807T					
+C27889T					
+A28271T					
+C28311T	N:P13L	ORF9b:P10S			This mutation causes ORG9b:P10S, but the adjacent mutation changes to P10F
+C28312T	N:P13L	ORF9b:P10F	y		This mutation on top of the one above, change ORf9b:P10S to P10F
+G28362-	N:E31-	ORF9b:E27-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+A28363-	N:E31-	ORF9b:E27-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+G28364-	N:E31-	ORF9b:E27-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+A28365-	N:R32-	ORF9b:N28-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+A28366-	N:R32-	ORF9b:N28-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+C28367-	N:R32-	ORF9b:N28-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+G28368-	N:S33-	ORF9b:A29-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+C28369-	N:S33-	ORF9b:A29-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+A28370-	N:S33-	ORF9b:A29-			Deletion in-frame for ORF9b; in N, deletes last 2 bases of AA 30, entirety of 31/32, first base of 33
+G28681T	N:E136D		y		
+G28881A	N:R203K				
+G28882A	N:R203K				
+G28883C	N:G204R				
+A29510C	N:S413R				
+G29734-					
+A29735-					
+G29736-					
+G29737-					
+C29738-					
+C29739-					
+A29740-					
+C29741-					
+G29742-					
+C29743-					
+G29744-					
+G29745-					
+A29746-					
+G29747-					
+T29748-					
+A29749-					
+C29750-					
+G29751-					
+A29752-					
+T29753-					
+C29754-					
+G29755-					
+A29756-					
+G29757-					
+T29758-					
+G29759-					
\ No newline at end of file
diff --git a/tests/test_merge_defining_mutations.py b/tests/test_merge_defining_mutations.py
new file mode 100644
index 0000000000..a3080f6354
--- /dev/null
+++ b/tests/test_merge_defining_mutations.py
@@ -0,0 +1,162 @@
+import json
+import os
+
+import numpy as np
+import pytest
+
+from scripts.merge_defining_mutations import process_auto_generated_data, process_hand_curated_file, \
+    match_nuc_to_aas, main, replace_list_of_empty_string, load_auto_generated_data, Mutation, \
+    AminoAcidMutation
+
+
+@pytest.mark.parametrize(
+    'nuc, nuc_dict',
+    [
+        ("T670G", {'symbol_from': 'T', 'position': 670, 'symbol_to': 'G'}),
+        ("C11289-", {'symbol_from': 'C', 'position': 11289, 'symbol_to': '-'}),
+        ("G10447A", {'symbol_from': 'G', 'position': 10447, 'symbol_to': 'A'}),
+        ("G4-", {'symbol_from': 'G', 'position': 4, 'symbol_to': '-'})
+    ]
+)
+def test_split_nuc(nuc, nuc_dict):
+    assert Mutation.parse_mutation_string(nuc).__dict__ == nuc_dict
+
+
+@pytest.mark.parametrize(
+    'nuc, genes',
+    [
+        ('T670G', ['ORF1a']),
+        ('C21636-', ['S']),
+        ('G28312T', ['N', 'ORF9b'])
+    ]
+)
+def test_nuc_to_genes(nuc, genes):
+    assert Mutation.parse_mutation_string(nuc).affected_genes == genes
+
+
+@pytest.mark.parametrize(
+    'aa, aa_tuple',
+    [
+        ('ORF9b:P10S', {'gene': 'ORF9b', 'symbol_from': 'P', 'position': 10, 'symbol_to': 'S'}),
+        ('S:P681H', {'gene': 'S', 'symbol_from':'P', 'position': 681, 'symbol_to': 'H'}),
+        ('ORF9b:N28-', {'gene': 'ORF9b', 'symbol_from': 'N', 'position':28, 'symbol_to':'-'})
+    ]
+)
+def test_split_aa(aa, aa_tuple):
+    assert AminoAcidMutation.parse_amino_acid_string(aa).__dict__ == aa_tuple
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+@pytest.mark.xfail(
+    reason='Nuc to aa matching algorithm needs to be checked: https://github.com/hodcroftlab/covariants/issues/578 ')
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    assert match_nuc_to_aas(nuc, aas) == 'S:H69-'
+
+
+@pytest.mark.xfail(
+    reason='need to clarify how we incorporate multiple amino acids: https://github.com/hodcroftlab/covariants/issues/581')
+def test_match_nuc_to_multiple_aas():
+    nuc = "C28312T"
+    aas = ['N:P13L', 'ORF9b:P10F']
+    assert match_nuc_to_aas(nuc, aas) == ['N:P13L', 'ORF9b:P10F']
+
+
+def test_remove_empty_strings():
+    assert replace_list_of_empty_string(['']) == []
+    assert replace_list_of_empty_string([]) == []
+    assert replace_list_of_empty_string(['hello']) == ['hello']
+    assert replace_list_of_empty_string(['hello', 'goodbye']) == ['hello', 'goodbye']
+    assert replace_list_of_empty_string('') == ''
+    assert replace_list_of_empty_string('hello') == 'hello'
+    assert replace_list_of_empty_string(np.array([''])) == []
+    assert replace_list_of_empty_string(np.array(['hello'])) == np.array(['hello'])
+
+
+def test_process_hand_curated_file():
+    hand_curated = process_hand_curated_file('tests/data/defining_mutations/hand_curated/22E.Omicron.tsv')
+    assert len(hand_curated) == 141
+    assert hand_curated.columns == ['nextstrain_clade', 'nuc_mutation', 'aa_mutation', 'reference', 'notes']
+
+
+def test_load_and_process_auto_generated_data():
+    lineages, auto_generated_mutations_raw = load_auto_generated_data(
+        'tests/data/defining_mutations/auto_generated/auto_generated.json')
+    auto_generated_mutations = process_auto_generated_data(auto_generated_mutations_raw)
+    assert len(lineages) == 2
+    assert lineages.columns == ['lineage',
+                                'unaliased',
+                                'parent',
+                                'children',
+                                'nextstrain_clade',
+                                'designation_date']
+    assert len(auto_generated_mutations) == 268
+    assert auto_generated_mutations.columns == ['lineage', 'nextstrain_clade', 'nuc_mutation', 'aa_mutation', 'reference']
+
+
+def test_main():
+    test_dir = 'tests/data/defining_mutations'
+    hand_curated_test_dir = os.path.join(test_dir, 'hand_curated')
+    auto_generated_test_dir = os.path.join(test_dir, 'auto_generated')
+    output_test_dir = os.path.join(test_dir, 'output')
+    expected_output_dir = os.path.join(test_dir, 'expected_output')
+    expected_output_filenames = os.listdir(expected_output_dir)
+
+    main(hand_curated_test_dir, auto_generated_test_dir, output_test_dir)
+
+    assert os.listdir(output_test_dir) == expected_output_filenames
+    expected_output_filenames.remove('definingMutationsClusters.json')
+    for filename in expected_output_filenames:
+        with open(os.path.join(output_test_dir, filename)) as output_file:
+            with open(os.path.join(expected_output_dir, filename)) as expected_output_file:
+                output = json.load(output_file)
+                expected_output = json.load(expected_output_file)
+                compare_output(expected_output, output)
+
+
+def compare_output(mutations1, mutations2):
+    assert mutations1['lineage'] == mutations2['lineage']
+    assert mutations1['nextstrain_clade'] == mutations2['nextstrain_clade']
+    assert mutations1['children'] == mutations2['children']
+    assert mutations1['designation_date'] == mutations2['designation_date']
+    assert mutations1['unaliased'] == mutations2['unaliased']
+    assert mutations1['parent'] == mutations2['parent']
+
+    for muts1 in mutations1['mutations']:
+        for muts2 in mutations2['mutations']:
+            if muts1['reference'] == muts2['reference']:
+                for mutation_type in ['coding', 'silent']:
+                    equal = False
+                    for mut1 in muts1[mutation_type]:
+                        equal = False
+                        for mut2 in muts2[mutation_type]:
+                            if mut1 == mut2:
+                                equal = True
+                                break
+                            else:
+                                equal = False
+                        assert equal
+                    assert equal
+
+
+
+def test_edge_cases_do_not_throw_errors():
+    _, auto_generated_mutations_raw = load_auto_generated_data(
+        'tests/data/defining_mutations/auto_generated/auto_generated_edge_cases.json')
+    process_auto_generated_data(auto_generated_mutations_raw)


Reviews:
user: login: JonasKellerer
body: Great work. We are a huge step further.

Most of my comments are regarding refactoring. It feels this is more script style. But it could benefit from some functions to make the intention clearer. Also some more type hints would be nice.

Some of my points are adressed by #572. Feel free to ignore them, but I would then suggest that we merge the other PR as fast as possible, so the build works.
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-04-03T14:55:28Z
user: login: AdvancedCodingMonkey
body: Tried to address your points, let me know if anything is still missing.
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-04T13:13:44Z
user: login: JonasKellerer
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T06:21:22Z
user: login: JonasKellerer
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T06:33:53Z
user: login: JonasKellerer
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T06:36:03Z
user: login: JonasKellerer
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T06:36:33Z
user: login: JonasKellerer
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T06:39:14Z
user: login: AdvancedCodingMonkey
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T11:36:41Z
user: login: JonasKellerer
body: Very nice. This brings us much further. Thank you also for all the refactorings.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-08T14:38:44Z
user: login: AdvancedCodingMonkey
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T07:44:33Z
user: login: AdvancedCodingMonkey
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T07:56:46Z

Review comments:
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Could you link a corresponding ticket here, so we can keep track of this TODO? 
created_at: 2025-04-03T09:29:52Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
+
+    return grouped_by_aa
+
+
+def reformat_df_to_dicts(merged, lineages):
+    aggregate_mutations = (
+        merged
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type')
+        .agg(pl.struct('aa_change', 'nuc_change', 'notes').alias('mutations'))
+    )
+
+    pivot_mutation_types = (
+        aggregate_mutations
+        .pivot('mutation_type', values='mutations')
+        .select('lineage', 'nextstrain_clade', 'relative_to', mutation_type=pl.struct('coding', 'silent'))
+    )
+
+    pivot_mutations = (
+        pivot_mutation_types
+        .pivot('relative_to', values='mutation_type')
+        .select('lineage', 'nextstrain_clade', mutations_relative_to=pl.struct('wuhan', 'pango_parent'))
+    )
+
+    add_lineages = pivot_mutations.join(lineages, on=['lineage', 'nextstrain_clade'])
+
+    output = add_lineages.select('lineage',
+                                 'nextstrain_clade',
+                                 'mutations_relative_to',
+                                 'children',
+                                 'designation_date',
+                                 'unaliased',
+                                 'parent')
+
+    output_dicts = output.to_dicts()
+    for lineage in output_dicts:
+        for parent, mutation_types in lineage['mutations_relative_to'].items():
+            if not mutation_types['silent']:
+                mutation_types['silent'] = []
+            for silent_mutation in mutation_types['silent']:
+                silent_mutation.pop('aa_change')
+                silent_mutation.update({'nuc_change': nuc_string_to_object(silent_mutation['nuc_change'][0])})
+                if not silent_mutation['notes']:
+                    silent_mutation.pop('notes')
+            if not mutation_types['coding']:
+                mutation_types['coding'] = []
+            for coding_mutation in mutation_types['coding']:
+                coding_mutation.update({'aa_change': aa_string_to_object(coding_mutation['aa_change'])})
+                coding_mutation.update(
+                    {'nuc_change': [nuc_string_to_object(mutation) for mutation in coding_mutation['nuc_change']]})
+                if not coding_mutation['notes']:
+                    coding_mutation.pop('notes')
+
+    return output_dicts
+
+
+def main(emma_dir='../tests/data/defining_mutations/emma', corn_dir='../tests/data/defining_mutations/cornelius',
+         output_dir='../tests/data/defining_mutations/output'):
+    lineages, emma, corn = import_file_dfs(emma_dir, corn_dir)
+
+    merged = merge_file_dfs(emma, corn)
+
+    output = reformat_df_to_dicts(merged, lineages)
+
+    for lineage in output:
+        with open(os.path.join(output_dir, f'{lineage["lineage"]}.json'), 'w') as f:
+            json.dump(lineage, f)
+    # TODO: nextclade parent
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Could you also link a ticket here?
created_at: 2025-04-03T09:30:09Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Is there a ticket we can link? Which cases do you mean?
created_at: 2025-04-03T09:35:40Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
def nuc_location_to_gene(nuc_location: int):
```
created_at: 2025-04-03T09:37:03Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: What does dfs abbreviate? Could we spell it out?
created_at: 2025-04-03T10:04:49Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
def import_file_dfs(emma_dir: str, corn_dir: str):
```
created_at: 2025-04-03T11:10:11Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Same comment with the ticket.
created_at: 2025-04-03T11:11:55Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Same comment with the ticket.
created_at: 2025-04-03T11:14:45Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
def replace_list_of_empty_string(list: List[str]) -> List[str]:
  return [] if list == [''] else list
```
Does this also work. If yes, I think it would convey the idea better.
created_at: 2025-04-03T11:22:36Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
lineages = rename.select(
    ['lineage',
     'unaliased',
     'parent',
     'children',
     'nextstrain_clade',
     'designation_date'])
```
The documentation suggests, this could also work, or is there a reason we are not using the same syntax as for the mutations?
created_at: 2025-04-03T11:28:02Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
def process_emma_file(path: str):
```
created_at: 2025-04-03T11:33:25Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: I would put this into a function, which returns the dataframe. Maybe named: `readData`.
created_at: 2025-04-03T11:51:55Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: I would then put this into a function called `getLineages`.
created_at: 2025-04-03T11:52:22Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: It would be nice, if this is in a function. When I see it correctly, then it converts the deletion ranges to ningle deletion entries?
created_at: 2025-04-03T11:54:19Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
        lambda deletion_column: [[int(pos) for pos in del_range.split('-')] for del_range in deletion_column])
```
created_at: 2025-04-03T11:54:37Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
        lambda deletion_ranges: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in deletion_ranges])
```
created_at: 2025-04-03T12:00:34Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: I think we can puth these one after the other
```suggestion
    split_deletions = empty_strings_removed[deletion_columns]
      .applymap(
        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
        .applymap(
        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
        .applymap(lambda x: list(pd.core.common.flatten(x)))
```
created_at: 2025-04-03T12:02:10Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
def parse_mutation(mutation: str) -> tuple[str, int, str]:
```
created_at: 2025-04-03T12:09:36Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: This only matches to one amino acid mutation. Could there be the case, that one nucleotide mutations maps to two amino acid mutations? I think there are two conflicting tests: `test_match_nuc_to_aas` and `test_match_nuc_to_multiple_aas`
created_at: 2025-04-03T13:34:46Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Is this really an error? Can there be the case that we have nucleotide mutations outside of genes?
created_at: 2025-04-03T13:37:00Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Just from looking at `GENES_BOUNDS`, positions before 266 could be it. 
created_at: 2025-04-03T13:37:52Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Are the last positions overlapping? Meaning 13468 is both in ORF1a and ORF1b?
created_at: 2025-04-03T13:38:29Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: It would be nice to have a dataclass for nucleotide mutations:

```
@dataclass
class NucleotideMutation:
   symbol_from: str
   position: int
   symbol_to: str

   @classmethod
    def parse(cls, mutation_string: str) -> "NucleotideMutation":
        ...
```
This would enable us also to get the values in a structured way. For example in `match_nuc_to_aas`

Similarly I would do that for amino acid mutations.
  
created_at: 2025-04-03T13:42:47Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: We could then also get rid of nuc_string_to_object and add this to the dataclass.
created_at: 2025-04-03T13:43:35Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Nitpick: In python the objects are called dicts. :)
created_at: 2025-04-03T13:44:14Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
    validate_relative_to_column_is_filled(output)
```
I was confused what this function does. Maybe the name transports the idea better. (I must admit that my python knowledge is a bit rusty.)
created_at: 2025-04-03T13:50:50Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: I think we can improve on the naming here :) I suggest: `mutations_derived_by_cornelius` or something similar. 
created_at: 2025-04-03T13:53:06Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: I think it would be nice to put this into a function.
created_at: 2025-04-03T13:53:57Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Also this name can be improved :)
created_at: 2025-04-03T13:54:17Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
    filename = os.path.basename(path)
    lineage = os.path.splitext(filename)[0]
```
created_at: 2025-04-03T13:56:50Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Maybe we could get rid of the people names entirely and use the tools name, which creates them. Do we know that?
created_at: 2025-04-03T13:57:40Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: What do we do with that info? Do we run it again, or should this be an error?
created_at: 2025-04-03T13:59:02Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: It would be nice to have this in a function. 
created_at: 2025-04-03T14:01:20Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: I am not sure if we need the `X` symbol here. It least for cov-spectrum, a mutation with `123-` is still a valid mutation. We would need to take this into account for the nucleotide mutations objects above. 
However, how I see it, this will end up in the output. `X` is no ambigous symbol for nucleotides but for amino acids.
created_at: 2025-04-03T14:11:15Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
+
+    return grouped_by_aa
+
+
+def reformat_df_to_dicts(merged, lineages):
+    aggregate_mutations = (
+        merged
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type')
+        .agg(pl.struct('aa_change', 'nuc_change', 'notes').alias('mutations'))
+    )
+
+    pivot_mutation_types = (
+        aggregate_mutations
+        .pivot('mutation_type', values='mutations')
+        .select('lineage', 'nextstrain_clade', 'relative_to', mutation_type=pl.struct('coding', 'silent'))
+    )
+
+    pivot_mutations = (
+        pivot_mutation_types
+        .pivot('relative_to', values='mutation_type')
+        .select('lineage', 'nextstrain_clade', mutations_relative_to=pl.struct('wuhan', 'pango_parent'))
+    )
+
+    add_lineages = pivot_mutations.join(lineages, on=['lineage', 'nextstrain_clade'])
+
+    output = add_lineages.select('lineage',
+                                 'nextstrain_clade',
+                                 'mutations_relative_to',
+                                 'children',
+                                 'designation_date',
+                                 'unaliased',
+                                 'parent')
+
+    output_dicts = output.to_dicts()
+    for lineage in output_dicts:
+        for parent, mutation_types in lineage['mutations_relative_to'].items():
+            if not mutation_types['silent']:
+                mutation_types['silent'] = []
+            for silent_mutation in mutation_types['silent']:
+                silent_mutation.pop('aa_change')
+                silent_mutation.update({'nuc_change': nuc_string_to_object(silent_mutation['nuc_change'][0])})
+                if not silent_mutation['notes']:
+                    silent_mutation.pop('notes')
+            if not mutation_types['coding']:
+                mutation_types['coding'] = []
+            for coding_mutation in mutation_types['coding']:
+                coding_mutation.update({'aa_change': aa_string_to_object(coding_mutation['aa_change'])})
+                coding_mutation.update(
+                    {'nuc_change': [nuc_string_to_object(mutation) for mutation in coding_mutation['nuc_change']]})
+                if not coding_mutation['notes']:
+                    coding_mutation.pop('notes')
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Could we do this in two steps, first we transform everything to a dict, then remove all empty fields? It is hard to find out if this function does what we want it to from looking at it. 
created_at: 2025-04-03T14:18:37Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
+
+    return grouped_by_aa
+
+
+def reformat_df_to_dicts(merged, lineages):
+    aggregate_mutations = (
+        merged
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type')
+        .agg(pl.struct('aa_change', 'nuc_change', 'notes').alias('mutations'))
+    )
+
+    pivot_mutation_types = (
+        aggregate_mutations
+        .pivot('mutation_type', values='mutations')
+        .select('lineage', 'nextstrain_clade', 'relative_to', mutation_type=pl.struct('coding', 'silent'))
+    )
+
+    pivot_mutations = (
+        pivot_mutation_types
+        .pivot('relative_to', values='mutation_type')
+        .select('lineage', 'nextstrain_clade', mutations_relative_to=pl.struct('wuhan', 'pango_parent'))
+    )
+
+    add_lineages = pivot_mutations.join(lineages, on=['lineage', 'nextstrain_clade'])
+
+    output = add_lineages.select('lineage',
+                                 'nextstrain_clade',
+                                 'mutations_relative_to',
+                                 'children',
+                                 'designation_date',
+                                 'unaliased',
+                                 'parent')
+
+    output_dicts = output.to_dicts()
+    for lineage in output_dicts:
+        for parent, mutation_types in lineage['mutations_relative_to'].items():
+            if not mutation_types['silent']:
+                mutation_types['silent'] = []
+            for silent_mutation in mutation_types['silent']:
+                silent_mutation.pop('aa_change')
+                silent_mutation.update({'nuc_change': nuc_string_to_object(silent_mutation['nuc_change'][0])})
+                if not silent_mutation['notes']:
+                    silent_mutation.pop('notes')
+            if not mutation_types['coding']:
+                mutation_types['coding'] = []
+            for coding_mutation in mutation_types['coding']:
+                coding_mutation.update({'aa_change': aa_string_to_object(coding_mutation['aa_change'])})
+                coding_mutation.update(
+                    {'nuc_change': [nuc_string_to_object(mutation) for mutation in coding_mutation['nuc_change']]})
+                if not coding_mutation['notes']:
+                    coding_mutation.pop('notes')
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: We could then also put both steps into its own function :)
created_at: 2025-04-03T14:19:02Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
+
+    return grouped_by_aa
+
+
+def reformat_df_to_dicts(merged, lineages):
+    aggregate_mutations = (
+        merged
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type')
+        .agg(pl.struct('aa_change', 'nuc_change', 'notes').alias('mutations'))
+    )
+
+    pivot_mutation_types = (
+        aggregate_mutations
+        .pivot('mutation_type', values='mutations')
+        .select('lineage', 'nextstrain_clade', 'relative_to', mutation_type=pl.struct('coding', 'silent'))
+    )
+
+    pivot_mutations = (
+        pivot_mutation_types
+        .pivot('relative_to', values='mutation_type')
+        .select('lineage', 'nextstrain_clade', mutations_relative_to=pl.struct('wuhan', 'pango_parent'))
+    )
+
+    add_lineages = pivot_mutations.join(lineages, on=['lineage', 'nextstrain_clade'])
+
+    output = add_lineages.select('lineage',
+                                 'nextstrain_clade',
+                                 'mutations_relative_to',
+                                 'children',
+                                 'designation_date',
+                                 'unaliased',
+                                 'parent')
+
+    output_dicts = output.to_dicts()
+    for lineage in output_dicts:
+        for parent, mutation_types in lineage['mutations_relative_to'].items():
+            if not mutation_types['silent']:
+                mutation_types['silent'] = []
+            for silent_mutation in mutation_types['silent']:
+                silent_mutation.pop('aa_change')
+                silent_mutation.update({'nuc_change': nuc_string_to_object(silent_mutation['nuc_change'][0])})
+                if not silent_mutation['notes']:
+                    silent_mutation.pop('notes')
+            if not mutation_types['coding']:
+                mutation_types['coding'] = []
+            for coding_mutation in mutation_types['coding']:
+                coding_mutation.update({'aa_change': aa_string_to_object(coding_mutation['aa_change'])})
+                coding_mutation.update(
+                    {'nuc_change': [nuc_string_to_object(mutation) for mutation in coding_mutation['nuc_change']]})
+                if not coding_mutation['notes']:
+                    coding_mutation.pop('notes')
+
+    return output_dicts
+
+
+def main(emma_dir='../tests/data/defining_mutations/emma', corn_dir='../tests/data/defining_mutations/cornelius',
+         output_dir='../tests/data/defining_mutations/output'):
+    lineages, emma, corn = import_file_dfs(emma_dir, corn_dir)
+
+    merged = merge_file_dfs(emma, corn)
+
+    output = reformat_df_to_dicts(merged, lineages)
+
+    for lineage in output:
+        with open(os.path.join(output_dir, f'{lineage["lineage"]}.json'), 'w') as f:
+            json.dump(lineage, f)
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: I think this could also be put into a function.
created_at: 2025-04-03T14:19:23Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
def match_nuc_to_aas(nuc: str, aas: list[str]) -> str | None:
```
created_at: 2025-04-03T14:23:08Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
        lambda row: match_nuc_to_aas(row['nuc_change'], row['aa_change']), pl.String
```
I'm not sure if this is a row, but it is a bit better than x
created_at: 2025-04-03T14:26:51Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: In the expected output there are no longer the `X` symbols, but weh I run the test, I get it in the output.
created_at: 2025-04-03T14:35:59Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    # TODO: this example is also classified differently in corn vs emma (wuhan vs pango_parent)
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: Same as above: Add a ticket.
created_at: 2025-04-03T14:36:58Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    # TODO: this example is also classified differently in corn vs emma (wuhan vs pango_parent)
+    assert match_nuc_to_aas(nuc, aas) == 'S:H69-'
+
+
+def test_match_nuc_to_multiple_aas():
+    nuc = "C28312T"
+    aas = ['N:P13L', 'ORF9b:P10F']
+    # TODO: need to clarify how we incorporate multiple amino acids
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: Same as above: Add a ticket.
created_at: 2025-04-03T14:37:01Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    # TODO: this example is also classified differently in corn vs emma (wuhan vs pango_parent)
+    assert match_nuc_to_aas(nuc, aas) == 'S:H69-'
+
+
+def test_match_nuc_to_multiple_aas():
+    nuc = "C28312T"
+    aas = ['N:P13L', 'ORF9b:P10F']
+    # TODO: need to clarify how we incorporate multiple amino acids
+    assert match_nuc_to_aas(nuc, aas) == ['N:P13L', 'ORF9b:P10F']
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: I think this testcase contradicts `test_match_missed_nuc_to_aas`, but maybe you meant this in the comment.
created_at: 2025-04-03T14:37:33Z
author_association: COLLABORATOR
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: I needed to add `pytest` and `pyarrow` to the requirements. But I have found this also in https://github.com/hodcroftlab/covariants/pull/572
created_at: 2025-04-03T14:38:48Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: I'm not sure how verbose we want to test, but we do not catch anything that is of the wrong format. So "ABC" is maybe also transformed, but I don't know :)
created_at: 2025-04-03T14:40:33Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: I think we can parameterize these tests. 
created_at: 2025-04-03T14:41:16Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: Same as above. 
created_at: 2025-04-03T14:41:31Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: Same as above.
created_at: 2025-04-03T14:41:39Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    # TODO: this example is also classified differently in corn vs emma (wuhan vs pango_parent)
+    assert match_nuc_to_aas(nuc, aas) == 'S:H69-'
+
+
+def test_match_nuc_to_multiple_aas():
+    nuc = "C28312T"
+    aas = ['N:P13L', 'ORF9b:P10F']
+    # TODO: need to clarify how we incorporate multiple amino acids
+    assert match_nuc_to_aas(nuc, aas) == ['N:P13L', 'ORF9b:P10F']
+
+
+def test_process_emma_file():
+    emma = process_emma_file('data/defining_mutations/emma/22E.Omicron.tsv')
+    assert len(emma) == 141
+    assert emma.columns == ['nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'notes']
+
+
+def test_process_cornelius_file():
+    lineages, corn = process_cornelius_file('data/defining_mutations/cornelius/cornelius.json')
+    assert len(lineages) == 2
+    assert lineages.columns == ['lineage',
+                                'unaliased',
+                                'parent',
+                                'children',
+                                'nextstrain_clade',
+                                'designation_date']
+    assert len(corn) == 258
+    assert corn.columns == ['lineage', 'nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to']
+
+
+def test_main():
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: This test fails for me.
created_at: 2025-04-03T14:43:28Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,518 @@
+{
+  "B": {
path: tests/data/defining_mutations/cornelius/cornelius_edge_cases.json
user: login: JonasKellerer
body: Could we name these edge cases somehow to describe what they should test?
created_at: 2025-04-03T14:44:47Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: ```suggestion
def process_cornelius_file(path: str):
```
created_at: 2025-04-03T14:45:40Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Hmm, I don't exactly know (and did not notice so far). My guess would be that it is a half-open interval, which would be consistent with python's `range()` function. Currently, we only use the start value, so we are interpreting the intervals as half-open. We'll probably have to revisit the nuc-to-aa matching after [clarification with Cornelius](https://github.com/hodcroftlab/covariants/issues/577) anyway.
created_at: 2025-04-04T07:57:06Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: I talked to Emma about it and it seems there are non-gene regions at the start and end. And then there can be gaps between the genes (which is not the case for many other viruses). So no, it is not actually an error.
created_at: 2025-04-04T08:06:14Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: No, it actually does not work because pandas is passing down numpy arrays here. I will leave it as it is because converting would make the function very verbose again.
created_at: 2025-04-04T09:04:35Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: No reason, I only found out the plain list works later on and forgot to change.
created_at: 2025-04-04T09:07:20Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: It is not actually a column but a cell (because the cells contain lists).
created_at: 2025-04-04T09:09:07Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: It is actually also a deletion_cell, the format of the data frame has not changed.
created_at: 2025-04-04T09:20:01Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: I purposefully did not chain them together for easier understanding and debugging. Looking at it now it makes things a lot cleaner though.
created_at: 2025-04-04T09:46:56Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: The second one would break the current matching with Cornelius' file but can be addressed when we take care of the unique identifier.
created_at: 2025-04-04T09:55:37Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Oh yes, I have been putting this off. I think hand-curated vs auto-generated fits well.
created_at: 2025-04-04T11:04:32Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: I think it is useful to know how many mutations coming from the hand-curated data are not matched in the auto-generated data. The number is really high and I am not quite convinced yet that it is how it is supposed to be. Initially, it was also useful for me to debug the pipeline. I will continue to log this for now.
created_at: 2025-04-04T11:21:44Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Since it is very cumbersome to define APIs between functions that use data frames, I would like to keep the data pipeline in one piece with small steps. Too often, a column gets lost in the handover and errors become hard to debug.
created_at: 2025-04-04T11:27:55Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
+
+    return grouped_by_aa
+
+
+def reformat_df_to_dicts(merged, lineages):
+    aggregate_mutations = (
+        merged
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type')
+        .agg(pl.struct('aa_change', 'nuc_change', 'notes').alias('mutations'))
+    )
+
+    pivot_mutation_types = (
+        aggregate_mutations
+        .pivot('mutation_type', values='mutations')
+        .select('lineage', 'nextstrain_clade', 'relative_to', mutation_type=pl.struct('coding', 'silent'))
+    )
+
+    pivot_mutations = (
+        pivot_mutation_types
+        .pivot('relative_to', values='mutation_type')
+        .select('lineage', 'nextstrain_clade', mutations_relative_to=pl.struct('wuhan', 'pango_parent'))
+    )
+
+    add_lineages = pivot_mutations.join(lineages, on=['lineage', 'nextstrain_clade'])
+
+    output = add_lineages.select('lineage',
+                                 'nextstrain_clade',
+                                 'mutations_relative_to',
+                                 'children',
+                                 'designation_date',
+                                 'unaliased',
+                                 'parent')
+
+    output_dicts = output.to_dicts()
+    for lineage in output_dicts:
+        for parent, mutation_types in lineage['mutations_relative_to'].items():
+            if not mutation_types['silent']:
+                mutation_types['silent'] = []
+            for silent_mutation in mutation_types['silent']:
+                silent_mutation.pop('aa_change')
+                silent_mutation.update({'nuc_change': nuc_string_to_object(silent_mutation['nuc_change'][0])})
+                if not silent_mutation['notes']:
+                    silent_mutation.pop('notes')
+            if not mutation_types['coding']:
+                mutation_types['coding'] = []
+            for coding_mutation in mutation_types['coding']:
+                coding_mutation.update({'aa_change': aa_string_to_object(coding_mutation['aa_change'])})
+                coding_mutation.update(
+                    {'nuc_change': [nuc_string_to_object(mutation) for mutation in coding_mutation['nuc_change']]})
+                if not coding_mutation['notes']:
+                    coding_mutation.pop('notes')
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: It is two steps :laughing: the long loop only removes the empty fields.
created_at: 2025-04-04T11:32:42Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
+
+    return grouped_by_aa
+
+
+def reformat_df_to_dicts(merged, lineages):
+    aggregate_mutations = (
+        merged
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type')
+        .agg(pl.struct('aa_change', 'nuc_change', 'notes').alias('mutations'))
+    )
+
+    pivot_mutation_types = (
+        aggregate_mutations
+        .pivot('mutation_type', values='mutations')
+        .select('lineage', 'nextstrain_clade', 'relative_to', mutation_type=pl.struct('coding', 'silent'))
+    )
+
+    pivot_mutations = (
+        pivot_mutation_types
+        .pivot('relative_to', values='mutation_type')
+        .select('lineage', 'nextstrain_clade', mutations_relative_to=pl.struct('wuhan', 'pango_parent'))
+    )
+
+    add_lineages = pivot_mutations.join(lineages, on=['lineage', 'nextstrain_clade'])
+
+    output = add_lineages.select('lineage',
+                                 'nextstrain_clade',
+                                 'mutations_relative_to',
+                                 'children',
+                                 'designation_date',
+                                 'unaliased',
+                                 'parent')
+
+    output_dicts = output.to_dicts()
+    for lineage in output_dicts:
+        for parent, mutation_types in lineage['mutations_relative_to'].items():
+            if not mutation_types['silent']:
+                mutation_types['silent'] = []
+            for silent_mutation in mutation_types['silent']:
+                silent_mutation.pop('aa_change')
+                silent_mutation.update({'nuc_change': nuc_string_to_object(silent_mutation['nuc_change'][0])})
+                if not silent_mutation['notes']:
+                    silent_mutation.pop('notes')
+            if not mutation_types['coding']:
+                mutation_types['coding'] = []
+            for coding_mutation in mutation_types['coding']:
+                coding_mutation.update({'aa_change': aa_string_to_object(coding_mutation['aa_change'])})
+                coding_mutation.update(
+                    {'nuc_change': [nuc_string_to_object(mutation) for mutation in coding_mutation['nuc_change']]})
+                if not coding_mutation['notes']:
+                    coding_mutation.pop('notes')
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: I have refactored it to make it easier to understand.
created_at: 2025-04-04T11:53:50Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Nice catch. We will handle this in #581 , I have annotated the test so this becomes clear.
created_at: 2025-04-04T12:05:40Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: I don't think this split makes sense here. It would be a very shallow function that has the same API as the parent. I think the split should be between loading and processing, such that the processing function gets the correct data frame as an input.
created_at: 2025-04-04T12:11:14Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Similar to my comment above, I think getting the lineages can be taken out of the processing function entirely.
created_at: 2025-04-04T12:11:32Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: I talked to Emma about it and we will inquire if we can get the nucleotide information for the auto-generated deletions: https://github.com/hodcroftlab/covariants/issues/577 . The `X`s get removed during the merge because all deletions are present in the hand-curated files. Only if a deletion is not found there, will the `X` propagate to the final output.
created_at: 2025-04-04T12:41:36Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: It expands the ranges (e.g. '12-15' to [12,13,14,15]). As I outlined in other comments, I want to keep the pipeline in one piece and with the refactoring, the whole expansion is only one step.
created_at: 2025-04-04T12:46:28Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Hmm, feels like over-engineering it at the moment.
created_at: 2025-04-04T12:48:03Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    # TODO: this example is also classified differently in corn vs emma (wuhan vs pango_parent)
+    assert match_nuc_to_aas(nuc, aas) == 'S:H69-'
+
+
+def test_match_nuc_to_multiple_aas():
+    nuc = "C28312T"
+    aas = ['N:P13L', 'ORF9b:P10F']
+    # TODO: need to clarify how we incorporate multiple amino acids
+    assert match_nuc_to_aas(nuc, aas) == ['N:P13L', 'ORF9b:P10F']
+
+
+def test_process_emma_file():
+    emma = process_emma_file('data/defining_mutations/emma/22E.Omicron.tsv')
+    assert len(emma) == 141
+    assert emma.columns == ['nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'notes']
+
+
+def test_process_cornelius_file():
+    lineages, corn = process_cornelius_file('data/defining_mutations/cornelius/cornelius.json')
+    assert len(lineages) == 2
+    assert lineages.columns == ['lineage',
+                                'unaliased',
+                                'parent',
+                                'children',
+                                'nextstrain_clade',
+                                'designation_date']
+    assert len(corn) == 258
+    assert corn.columns == ['lineage', 'nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to']
+
+
+def test_main():
path: tests/test_merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: It should not. Are you sure everything is correctly set up with the working directory?
created_at: 2025-04-04T12:49:58Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    # TODO: this example is also classified differently in corn vs emma (wuhan vs pango_parent)
+    assert match_nuc_to_aas(nuc, aas) == 'S:H69-'
+
+
+def test_match_nuc_to_multiple_aas():
+    nuc = "C28312T"
+    aas = ['N:P13L', 'ORF9b:P10F']
+    # TODO: need to clarify how we incorporate multiple amino acids
+    assert match_nuc_to_aas(nuc, aas) == ['N:P13L', 'ORF9b:P10F']
path: tests/test_merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Exactly, I hope it is clearer now
created_at: 2025-04-04T13:04:27Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
path: tests/test_merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: I think we are ok for the moment :)
created_at: 2025-04-04T13:06:00Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Okay. I think we should name it differently, so it is clear that it only replaces empty strings in the arrays.

Also a nitpick: I think 
```
if len(y) == 1 and y[0] == '':
```
will do the same. 
created_at: 2025-04-07T06:21:22Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: We can wait for the clarification in #577 .
created_at: 2025-04-07T06:33:53Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
+    return nuc[0], int(nuc[1:-1]), nuc[-1]
+
+
+def split_aa(aa: str) -> tuple[str, str, int, str]:
+    gene, aa = aa.split(':')
+    return gene, *split_nuc(aa)
+
+
+def nuc_to_gene(nuc_location: int):
+    genes = []
+    for gene, bounds in GENE_BOUNDS.items():
+        if bounds[0] <= nuc_location <= bounds[1]:
+            genes.append(gene)
+    return genes
+
+
+def match_nuc_to_aas(nuc, aas):
+    if not nuc:
+        return
+    nuc_old, nuc_location, nuc_new = split_nuc(nuc)
+    genes = nuc_to_gene(nuc_location)
+    if len(genes) == 0:
+        print(f'WARNING: nuc {nuc} cannot be matched to gene region')
+        return
+    # TODO: this does not match correctly for all cases right now
+    locations = [int(np.floor((nuc_location - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in genes]
+    for aa in aas:
+        gene, aa_old, aa_location, aa_new = split_aa(aa)
+        if gene in genes and aa_location in locations:
+            return aa
+
+
+def nuc_string_to_object(nuc):
+    ref, pos, alt = split_nuc(nuc)
+    return {'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def aa_string_to_object(aa):
+    gene, ref, pos, alt = split_aa(aa)
+    return {'gene': gene, 'ref': ref, 'pos': pos, 'alt': alt}
+
+
+def remove_empty_strings(y):
+    if len(y) == 1:
+        if y[0] == '':
+            return []
+    return y
+
+
+def process_cornelius_file(path):
+    with open(path) as f:
+        data = json.load(f)
+
+    raw_lineages = [mut for mut in data.values()]
+
+    df = pl.from_records(raw_lineages)
+
+    rename = df.rename(
+        {'nextstrainClade': 'nextstrain_clade',
+         'designationDate': 'designation_date',
+         'nucSubstitutions': 'nuc_sub_wuhan',
+         'nucDeletions': 'nuc_del_wuhan',
+         'nucSubstitutionsNew': 'nuc_sub_pango_parent',
+         'nucDeletionsNew': 'nuc_del_pango_parent',
+         'aaSubstitutions': 'aa_sub_wuhan',
+         'aaDeletions': 'aa_del_wuhan',
+         'aaSubstitutionsNew': 'aa_sub_pango_parent',
+         'aaDeletionsNew': 'aa_del_pango_parent'}
+    )
+
+    lineages = rename.select(
+        pl.col('lineage'),
+        pl.col('unaliased'),
+        pl.col('parent'),
+        pl.col('children'),
+        pl.col('nextstrain_clade'),
+        pl.col('designation_date'))
+
+    mutations = rename.select(
+        'lineage',
+        'nextstrain_clade',
+        'nuc_sub_wuhan',
+        'nuc_del_wuhan',
+        'nuc_sub_pango_parent',
+        'nuc_del_pango_parent',
+        'aa_sub_wuhan',
+        'aa_del_wuhan',
+        'aa_sub_pango_parent',
+        'aa_del_pango_parent'
+    )
+
+    empty_strings_removed = mutations.to_pandas().applymap(remove_empty_strings)
+
+    deletion_columns = ['nuc_del_wuhan', 'nuc_del_pango_parent']
+    split_deletions = empty_strings_removed[deletion_columns].applymap(
+        lambda x: [[int(pos) for pos in del_range.split('-')] for del_range in x])
+    deletions_to_ranges = split_deletions.applymap(
+        lambda x: [list(range(*del_range)) if len(del_range) > 1 else del_range for del_range in x])
+    flatten_deletions = deletions_to_ranges.applymap(lambda x: list(pd.core.common.flatten(x)))
+    expand_deletions = empty_strings_removed.copy()
+    expand_deletions[deletion_columns] = flatten_deletions
+
+    reformat_deletions = pl.from_pandas(expand_deletions).with_columns(
+        pl.col(col_name).list.eval(
+            pl.concat_str(
+                pl.lit('X'),
+                pl.element(),
+                pl.lit('-'))
+        ) for col_name in deletion_columns
+    )
+
+    combine_nuc_changes = reformat_deletions.with_columns(
+        nuc_change_wuhan=pl.concat_list(
+            'nuc_sub_wuhan',
+            'nuc_del_wuhan'),
+        nuc_change_pango_parent=pl.concat_list(
+            'nuc_sub_pango_parent',
+            'nuc_del_pango_parent'
+        ),
+        aa_change_wuhan=pl.concat_list(
+            'aa_sub_wuhan',
+            'aa_del_wuhan'
+        ),
+        aa_change_pango_parent=pl.concat_list(
+            'aa_sub_pango_parent',
+            'aa_del_pango_parent'
+        )
+    )
+
+    # TODO: make sure the nuc to aa matching is scientifically correct
+    match_aa_changes = pl.struct('nuc_change', 'aa_change').map_elements(
+        lambda x: match_nuc_to_aas(x['nuc_change'], x['aa_change']), pl.String
+    )
+    wuhan = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change=pl.col('nuc_change_wuhan'),
+            aa_change=pl.col('aa_change_wuhan'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('wuhan')
+        )
+    )
+    pango_parent = (
+        combine_nuc_changes
+        .select(
+            pl.col('lineage'),
+            pl.col('nextstrain_clade'),
+            nuc_change='nuc_change_pango_parent',
+            aa_change=pl.col('aa_change_pango_parent'))
+        .explode('nuc_change')
+        .with_columns(
+            aa_change=match_aa_changes,
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([wuhan, pango_parent])
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return lineages, output
+
+
+def process_emma_file(path):
+    filename = path.split('/')[-1]
+    lineage = filename.split('.')[0]
+
+    data = pl.read_csv(path, separator='\t')
+
+    no_reversions = data.filter(pl.col('reversion').is_null())
+
+    with_lineage = (
+        no_reversions
+        .with_columns(
+            nextstrain_clade=pl.lit(lineage),
+            relative_to=pl.lit('wuhan')
+        )
+        .select('nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'not_in_parent', 'notes')
+    )
+    pango_parent = (
+        with_lineage.filter(pl.col('not_in_parent') == 'y')
+        .with_columns(
+            relative_to=pl.lit('pango_parent'))
+    )
+
+    output = pl.concat([with_lineage, pango_parent]).drop('not_in_parent')
+
+    check_relative_to_column(output)
+
+    # TODO: reversions, frame shifts
+    return output
+
+
+def check_relative_to_column(df):
+    if not df.filter(pl.col('relative_to').is_null()).is_empty():
+        raise ValueError(f'Could not assign reference point for mutations {df.filter(pl.col("relative_to").is_null())}')
+
+
+def import_file_dfs(emma_dir, corn_dir):
+    lineages, corn = process_cornelius_file(os.path.join(corn_dir, 'cornelius.json'))
+
+    emma_files = os.listdir(emma_dir)
+    emma_processed = []
+    for emma_file in emma_files:
+        emma_processed.append(
+            process_emma_file(os.path.join(emma_dir, emma_file))
+            .join(lineages
+                  .select('lineage', 'nextstrain_clade'), on='nextstrain_clade')
+        )
+    emma = pl.concat(emma_processed)
+
+    return lineages, emma, corn
+
+
+def merge_file_dfs(emma_df: pl.DataFrame, corn_df: pl.DataFrame):
+    emma_raw = emma_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    corn_raw = corn_df.with_columns(nuc_change_raw=pl.col('nuc_change').str.slice(1))
+    combined = (corn_raw
+                .join(emma_raw,
+                      on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'],
+                      how='full',
+                      suffix='_emma',
+                      coalesce=True)
+                .drop('nuc_change_raw'))
+
+    not_in_emma = len(
+        corn_raw.join(emma_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    not_in_corn = len(
+        emma_raw.join(corn_raw, on=['lineage', 'nextstrain_clade', 'nuc_change_raw', 'relative_to'], how='anti'))
+    if not_in_emma > 0 or not_in_corn > 0:
+        print(f'INFO: unmatched mutations: Emma {not_in_corn}, Cornelius {not_in_emma}')
+
+    coalesced = (
+        combined
+        .with_columns(
+            aa_change=pl.coalesce(pl.col('aa_change_emma'), pl.col('aa_change')),
+            nuc_change=pl.coalesce(pl.col('nuc_change_emma'), pl.col('nuc_change')))
+        .drop('aa_change_emma', 'nuc_change_emma')
+    )
+
+    typed = coalesced.with_columns(
+        mutation_type=pl.when(pl.col('aa_change').is_not_null()).then(pl.lit('coding')).otherwise(pl.lit('silent'))
+    ).select('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change', 'nuc_change', 'notes')
+
+    coding_grouped_by_aa = (
+        typed
+        .filter(pl.col('mutation_type') == 'coding')
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type', 'aa_change')
+        .agg(pl.col('nuc_change'), pl.col('notes').first())
+    )
+    silent_grouped_by_aa = typed.filter(pl.col('mutation_type') == 'silent').with_columns(
+        pl.col('nuc_change').cast(pl.List(pl.String)))
+    grouped_by_aa = pl.concat([
+        coding_grouped_by_aa,
+        silent_grouped_by_aa
+    ])
+
+    return grouped_by_aa
+
+
+def reformat_df_to_dicts(merged, lineages):
+    aggregate_mutations = (
+        merged
+        .group_by('lineage', 'nextstrain_clade', 'relative_to', 'mutation_type')
+        .agg(pl.struct('aa_change', 'nuc_change', 'notes').alias('mutations'))
+    )
+
+    pivot_mutation_types = (
+        aggregate_mutations
+        .pivot('mutation_type', values='mutations')
+        .select('lineage', 'nextstrain_clade', 'relative_to', mutation_type=pl.struct('coding', 'silent'))
+    )
+
+    pivot_mutations = (
+        pivot_mutation_types
+        .pivot('relative_to', values='mutation_type')
+        .select('lineage', 'nextstrain_clade', mutations_relative_to=pl.struct('wuhan', 'pango_parent'))
+    )
+
+    add_lineages = pivot_mutations.join(lineages, on=['lineage', 'nextstrain_clade'])
+
+    output = add_lineages.select('lineage',
+                                 'nextstrain_clade',
+                                 'mutations_relative_to',
+                                 'children',
+                                 'designation_date',
+                                 'unaliased',
+                                 'parent')
+
+    output_dicts = output.to_dicts()
+    for lineage in output_dicts:
+        for parent, mutation_types in lineage['mutations_relative_to'].items():
+            if not mutation_types['silent']:
+                mutation_types['silent'] = []
+            for silent_mutation in mutation_types['silent']:
+                silent_mutation.pop('aa_change')
+                silent_mutation.update({'nuc_change': nuc_string_to_object(silent_mutation['nuc_change'][0])})
+                if not silent_mutation['notes']:
+                    silent_mutation.pop('notes')
+            if not mutation_types['coding']:
+                mutation_types['coding'] = []
+            for coding_mutation in mutation_types['coding']:
+                coding_mutation.update({'aa_change': aa_string_to_object(coding_mutation['aa_change'])})
+                coding_mutation.update(
+                    {'nuc_change': [nuc_string_to_object(mutation) for mutation in coding_mutation['nuc_change']]})
+                if not coding_mutation['notes']:
+                    coding_mutation.pop('notes')
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Thank you. Much better now :)
created_at: 2025-04-07T06:36:03Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    # TODO: this example is also classified differently in corn vs emma (wuhan vs pango_parent)
+    assert match_nuc_to_aas(nuc, aas) == 'S:H69-'
+
+
+def test_match_nuc_to_multiple_aas():
+    nuc = "C28312T"
+    aas = ['N:P13L', 'ORF9b:P10F']
+    # TODO: need to clarify how we incorporate multiple amino acids
+    assert match_nuc_to_aas(nuc, aas) == ['N:P13L', 'ORF9b:P10F']
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: Very nice. 
created_at: 2025-04-07T06:36:33Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,119 @@
+import json
+import os
+
+from scripts.merge_defining_mutations import process_cornelius_file, process_emma_file, split_nuc, nuc_to_gene, \
+    split_aa, match_nuc_to_aas, main
+
+
+def test_split_nuc():
+    nuc1 = "T670G"
+    nuc2 = "C11289-"
+    nuc3 = "G10447A"
+    nuc4 = "G4-"
+    assert split_nuc(nuc1) == ('T', 670, 'G')
+    assert split_nuc(nuc2) == ('C', 11289, '-')
+    assert split_nuc(nuc3) == ('G', 10447, 'A')
+    assert split_nuc(nuc4) == ('G', 4, '-')
+
+
+def test_nuc_to_gene():
+    nuc1 = 670
+    nuc2 = 21636
+    nuc3 = 28312
+    assert nuc_to_gene(nuc1) == ['ORF1a']
+    assert nuc_to_gene(nuc2) == ['S']
+    assert nuc_to_gene(nuc3) == ['N', 'ORF9b']
+
+
+def test_split_aa():
+    aa1 = 'ORF9b:P10S'
+    aa2 = 'S:P681H'
+    aa3 = 'ORF9b:N28-'
+    assert split_aa(aa1) == ('ORF9b', 'P', 10, 'S')
+    assert split_aa(aa2) == ('S', 'P', 681, 'H')
+    assert split_aa(aa3) == ('ORF9b', 'N', 28, '-')
+
+
+def test_match_nuc_to_aas():
+    nuc = "T670G"
+    aas = [
+        "E:T9I",
+        "M:D3N",
+        "N:S413R",
+        "ORF1a:S135R",
+        "ORF1a:Q556K",
+        "ORF1b:T2163I",
+        "S:S373P",
+        "S:S375F",
+        "S:T376A",
+    ]
+    assert match_nuc_to_aas(nuc, aas) == 'ORF1a:S135R'
+
+
+def test_match_missed_nuc_to_aas():
+    nuc = "X21766-"
+    aas = ['S:H69-']
+    # TODO: this example is also classified differently in corn vs emma (wuhan vs pango_parent)
+    assert match_nuc_to_aas(nuc, aas) == 'S:H69-'
+
+
+def test_match_nuc_to_multiple_aas():
+    nuc = "C28312T"
+    aas = ['N:P13L', 'ORF9b:P10F']
+    # TODO: need to clarify how we incorporate multiple amino acids
+    assert match_nuc_to_aas(nuc, aas) == ['N:P13L', 'ORF9b:P10F']
+
+
+def test_process_emma_file():
+    emma = process_emma_file('data/defining_mutations/emma/22E.Omicron.tsv')
+    assert len(emma) == 141
+    assert emma.columns == ['nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to', 'notes']
+
+
+def test_process_cornelius_file():
+    lineages, corn = process_cornelius_file('data/defining_mutations/cornelius/cornelius.json')
+    assert len(lineages) == 2
+    assert lineages.columns == ['lineage',
+                                'unaliased',
+                                'parent',
+                                'children',
+                                'nextstrain_clade',
+                                'designation_date']
+    assert len(corn) == 258
+    assert corn.columns == ['lineage', 'nextstrain_clade', 'nuc_change', 'aa_change', 'relative_to']
+
+
+def test_main():
path: tests/test_merge_defining_mutations.py
user: login: JonasKellerer
body: Hm... I'm not sure what changed, but now it works :)
created_at: 2025-04-07T06:39:14Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,355 @@
+import json
+import os
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+def split_nuc(nuc: str) -> tuple[str, int, str]:
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Added data classes now, thanks for the push
created_at: 2025-04-07T11:36:41Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,390 @@
+import json
+import logging
+import os
+from dataclasses import dataclass
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+@dataclass
+class Mutation:
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Nitpick: This is only a mutation for covid. I general, all mutations have a coresponding segment. But since we only have covid, I think this is fine. And it is much easier this way. 

So the structure would be:
Mutation(segment, from, position, to)
NucleotideMutation(segment=null,from, position, to) : Mutation
AminoAcidMutation(segment=gene,from,position,to): Mutation
created_at: 2025-04-08T14:19:58Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,390 @@
+import json
+import logging
+import os
+from dataclasses import dataclass
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+@dataclass
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: I like the new data classes 😄 
created_at: 2025-04-08T14:22:17Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,390 @@
+import json
+import logging
+import os
+from dataclasses import dataclass
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+@dataclass
+class Mutation:
+    symbol_from: str
+    position: int
+    symbol_to: str
+
+    @classmethod
+    def parse_mutation_string(cls, mutation_string: str) -> "Mutation":
+        return Mutation(symbol_from=mutation_string[0], position=int(mutation_string[1:-1]), symbol_to=mutation_string[-1])
+
+    @property
+    def affected_genes(self):
+        genes = []
+        for gene, bounds in GENE_BOUNDS.items():
+            if bounds[0] <= self.position <= bounds[1]:
+                genes.append(gene)
+        return genes
+
+    @property
+    def positions_on_genes(self):
+        return [int(np.floor((self.position - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in self.affected_genes]
+
+    def to_dict(self):
+        return {'ref': self.symbol_from, 'pos': self.position, 'alt': self.symbol_to}
+
+
+
+@dataclass
+class AminoAcidMutation(Mutation):
+    gene: str
+
+    @classmethod
+    def parse_amino_acid_string(cls, amino_acid_mutations_string: str) -> "AminoAcidMutation":
+        gene, aa =amino_acid_mutations_string.split(':')
+        mut = cls.parse_mutation_string(aa)
+        return AminoAcidMutation(mut.symbol_from, mut.position, mut.symbol_to, gene)
+
+    def to_string(self) -> str:
path: scripts/merge_defining_mutations.py
user: login: JonasKellerer
body: Uninformed nitpick: Isn't this the `__str__` function, which we could overwrite?

In covSpectrum we named this `mutationCode` or only `code`. But this is also good. 
created_at: 2025-04-08T14:28:39Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,390 @@
+import json
+import logging
+import os
+from dataclasses import dataclass
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+@dataclass
+class Mutation:
+    symbol_from: str
+    position: int
+    symbol_to: str
+
+    @classmethod
+    def parse_mutation_string(cls, mutation_string: str) -> "Mutation":
+        return Mutation(symbol_from=mutation_string[0], position=int(mutation_string[1:-1]), symbol_to=mutation_string[-1])
+
+    @property
+    def affected_genes(self):
+        genes = []
+        for gene, bounds in GENE_BOUNDS.items():
+            if bounds[0] <= self.position <= bounds[1]:
+                genes.append(gene)
+        return genes
+
+    @property
+    def positions_on_genes(self):
+        return [int(np.floor((self.position - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in self.affected_genes]
+
+    def to_dict(self):
+        return {'ref': self.symbol_from, 'pos': self.position, 'alt': self.symbol_to}
+
+
+
+@dataclass
+class AminoAcidMutation(Mutation):
+    gene: str
+
+    @classmethod
+    def parse_amino_acid_string(cls, amino_acid_mutations_string: str) -> "AminoAcidMutation":
+        gene, aa =amino_acid_mutations_string.split(':')
+        mut = cls.parse_mutation_string(aa)
+        return AminoAcidMutation(mut.symbol_from, mut.position, mut.symbol_to, gene)
+
+    def to_string(self) -> str:
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: Yes, we could use `__str__`. I like having a bit of a difference here though, since `__str__` gives you a bit more information about the object, like the class name and the field names. Naming it code seems helpful.
created_at: 2025-04-10T07:44:33Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,390 @@
+import json
+import logging
+import os
+from dataclasses import dataclass
+
+import numpy as np
+import polars as pl
+import pandas as pd
+
+GENE_BOUNDS = {
+    'E': [26245, 26472],
+    'M': [26523, 27191],
+    'N': [28274, 29533],
+    'ORF1a': [266, 13468],
+    'ORF1b': [13468, 21555],
+    'ORF3a': [25393, 26220],
+    'ORF6': [27202, 27387],
+    'ORF7a': [27394, 27759],
+    'ORF7b': [27756, 27887],
+    'ORF8': [27894, 28259],
+    'ORF9b': [28284, 28577],
+    'S': [21563, 25384]
+}
+
+
+@dataclass
+class Mutation:
+    symbol_from: str
+    position: int
+    symbol_to: str
+
+    @classmethod
+    def parse_mutation_string(cls, mutation_string: str) -> "Mutation":
+        return Mutation(symbol_from=mutation_string[0], position=int(mutation_string[1:-1]), symbol_to=mutation_string[-1])
+
+    @property
+    def affected_genes(self):
+        genes = []
+        for gene, bounds in GENE_BOUNDS.items():
+            if bounds[0] <= self.position <= bounds[1]:
+                genes.append(gene)
+        return genes
+
+    @property
+    def positions_on_genes(self):
+        return [int(np.floor((self.position - GENE_BOUNDS[gene][0]) / 3)) + 1 for gene in self.affected_genes]
+
+    def to_dict(self):
+        return {'ref': self.symbol_from, 'pos': self.position, 'alt': self.symbol_to}
+
+
+
+@dataclass
+class AminoAcidMutation(Mutation):
+    gene: str
+
+    @classmethod
+    def parse_amino_acid_string(cls, amino_acid_mutations_string: str) -> "AminoAcidMutation":
+        gene, aa =amino_acid_mutations_string.split(':')
+        mut = cls.parse_mutation_string(aa)
+        return AminoAcidMutation(mut.symbol_from, mut.position, mut.symbol_to, gene)
+
+    def to_string(self) -> str:
path: scripts/merge_defining_mutations.py
user: login: AdvancedCodingMonkey
body: I am going to do it on the other branch though, then I don't have to rebase :)
created_at: 2025-04-10T07:56:46Z
author_association: COLLABORATOR

PR comments:
user: login: JonasKellerer
created_at: 2025-04-03T16:02:10Z
body: I just noticed something: We in the format of the defining mutations, we have `aa_change` and `nuc_change`. Speaking from the domain language then these would be `aa_mutations` and `nuc_mutations`. 
author_association: COLLABORATOR
user: login: JonasKellerer
created_at: 2025-04-03T16:03:19Z
body: Also what do you think about this structure of the json file? This would make it easier to add more references later on. It also gets rid of the `mutations_relative_to`.

```json
"mutations": [
    {
      "reference": "wuhan",
      "coding_mutations": [
        {
          "aa_mutation": {
            "gene": "ORF1a",
            "ref": "S",
            "pos": 135,
            "alt": "R"
          },
          "nuc_mutations": [
            {
              "ref": "T",
              "pos": 670,
              "alt": "G"
            }
          ]
        }
      ],
      "silent_mutations": [
        {
          "nuc_mutation": {
            "ref": "C",
            "pos": 241,
            "alt": "T"
          }
        }
      ]
    }
  ]
```

author_association: COLLABORATOR
user: login: JonasKellerer
created_at: 2025-04-04T10:36:15Z
body: I just added a ticket, to keep track of the link between the defining mutations and the clusters.json: https://github.com/hodcroftlab/covariants/issues/580
author_association: COLLABORATOR
user: login: JonasKellerer
created_at: 2025-04-08T14:48:55Z
body: Remember squash before merge :)
author_association: COLLABORATOR