Repository: Clinical-Genomics/scout

Issue:
CSV - length -1 for a deletion of 80 genes
https://scout.scilifelab.se/cust003/17159/sv/variants?variant_type=clinical&gene_panels=EP&hgnc_symbols=&size=&chrom=&thousand_genomes_frequency=

please have a look. Something doesn't fit here.
A deletion of 1 bp can not contain 80 genes.

thanks,
Michela

Comments:

Patch:
diff --git a/scout/build/panel.py b/scout/build/panel.py
index 1a4dfa0fd..85b02bfef 100644
--- a/scout/build/panel.py
+++ b/scout/build/panel.py
@@ -44,7 +44,7 @@ def build_gene(gene_info, adapter):
     
     hgnc_gene = adapter.hgnc_gene(hgnc_id)
     if hgnc_gene is None:
-        raise IntegrityError
+        raise IntegrityError("hgnc_id {0} is not in the gene database!".format(hgnc_id))
     
     gene_obj['symbol'] = hgnc_gene['hgnc_symbol']
 
diff --git a/scout/commands/load/panel.py b/scout/commands/load/panel.py
index 1aa029868..0569e2e13 100644
--- a/scout/commands/load/panel.py
+++ b/scout/commands/load/panel.py
@@ -74,5 +74,5 @@ def panel(context, date, name, version, panel_type, panel_id, path, institute):
     try:
         adapter.load_panel(info)
     except IntegrityError as e:
-        logger.warning(e)
+        log.warning(e)
         context.abort()
diff --git a/scout/parse/variant/coordinates.py b/scout/parse/variant/coordinates.py
index fa24acf8c..5031d6b0a 100644
--- a/scout/parse/variant/coordinates.py
+++ b/scout/parse/variant/coordinates.py
@@ -19,7 +19,20 @@ def get_cytoband_coordinates(chrom, pos):
     return coordinate
 
 def get_sub_category(alt_len, ref_len, category, svtype=None):
-    """Get the subcategory"""
+    """Get the subcategory for a VCF variant
+    
+    The sub categories are:
+        'snv', 'indel', 'del', 'ins', 'dup', 'bnd', 'inv'
+    
+    Args:
+        alt_len(int)
+        ref_len(int)
+        category(str)
+        svtype(str)
+    
+    Returns:
+        subcategory(str)
+    """
     subcategory = ''
     
     if category in ('snv', 'indel', 'cancer'):
@@ -32,99 +45,147 @@ def get_sub_category(alt_len, ref_len, category, svtype=None):
     
     return subcategory
 
-def get_length(alt_len, ref_len, category, svtype=None, svlen=None):
-    """docstring for get_length"""
+def get_length(alt_len, ref_len, category, pos, end, svtype=None, svlen=None):
+    """Return the length of a variant
+    
+    Args:
+        alt_len(int)
+        ref_len(int)
+        category(str)
+        svtype(str)
+        svlen(int)
+    """
+    # -1 would indicate uncertain length
+    length = -1
     if category in ('snv', 'indel', 'cancer'):
         if ref_len == alt_len:
             length = alt_len
         else:
-            length = abs(ref_len-alt_len)
+            length = abs(ref_len - alt_len)
+    
     elif category == 'sv':
         if svtype == 'bnd':
             length = int(10e10)
         else:
             if svlen:
                 length = abs(int(svlen))
-            else:
-                # -1 would indicate uncertain length
-                length = -1
+            # Some software does not give a length but they give END
+            elif end:
+                if end != pos:
+                    length = end - pos
     return length
 
-def get_end(pos, length, alt, category, svtype=None):
-    """docstring for get_length"""
-    end = None
+def get_end(pos, alt, category, snvend=None, svend=None, svlen=None):
+    """Return the end coordinate for a variant
+    
+    Args:
+        pos(int)
+        alt(str)
+        category(str)
+        snvend(str)
+        svend(int)
+        svlen(int)
+    
+    Returns:
+        end(int)
+    """
+    # If nothing is known we set end to be same as start
+    end = pos
+    # If variant is snv or indel we know that cyvcf2 can handle end pos
     if category in ('snv', 'indel', 'cancer'):
-        end = pos + length
+        end = snvend
 
+    # With SVs we have to be a bit more careful
     elif category == 'sv':
+        # The END field from INFO usually works fine
+        end = svend
+        
+        # For some cases like insertions the callers set end to same as pos
+        # In those cases we can hope that there is a svlen...
+        if svend == pos:
+            if svlen:
+                end = pos + svlen
+        # If variant is 'BND' they have ':' in alt field
+        # Information about other end is in the alt field
         if ':' in alt:
             other_coordinates = alt.strip('ACGTN[]').split(':')
             # For BND end will represent the end position of the other end
             try:
                 end = int(other_coordinates[1])
             except ValueError as err:
-                end = pos + length
-        else:
-            end = pos + length
-    
-    return end
+                pass
 
+    return end
 
-def parse_coordinates(chrom, ref, alt, position, category, svtype, svlen, end, mate_id=None):
+def parse_coordinates(variant, category):
     """Find out the coordinates for a variant
     
     Args:
-        chrom(str)
-        ref(str)
-        alt(str)
-        position(int)
-        category(str)
-        svtype(str)
-        svlen(int)
-        end(int)
-        mate_id(str)
+        variant(cyvcf2.Variant)
     
     Returns:
         coordinates(dict): A dictionary on the form:
         {
+            'position':<int>, 
             'end':<int>, 
+            'end_chrom':<str>,
             'length':<int>, 
             'sub_category':<str>,
             'mate_id':<str>,
+            'cytoband_start':<str>,
+            'cytoband_end':<str>,
         }
     """
-    coordinates = {
-        'end': end,
-        'length': None,
-        'sub_category': None,
-        'mate_id':None,
-        'cytoband_start':None,
-        'cytoband_end':None,
-        'end_chrom':None,
-    }
+    ref = variant.REF
+    alt = variant.ALT[0]
+    chrom = variant.CHROM
+    if (chrom.startswith('chr') or chrom.startswith('CHR')):
+        chrom = chrom[3:]
+    
+    svtype = variant.INFO.get('SVTYPE')
     if svtype:
         svtype = svtype.lower()
 
+    mate_id = variant.INFO.get('MATEID')
+
+    svlen = variant.INFO.get('SVLEN')
+    
+    svend = variant.INFO.get('END')
+    snvend = int(variant.end)
+    
+    position = int(variant.POS)
+
     ref_len = len(ref)
     alt_len = len(alt)
-    coordinates['mate_id'] = mate_id
-    coordinates['sub_category'] = get_sub_category(alt_len, ref_len, category, svtype)
-    coordinates['length'] = get_length(alt_len, ref_len, category, svtype, svlen)
-    coordinates['end'] = get_end(position, coordinates['length'], alt, category, svtype)
-    coordinates['end_chrom'] = chrom
-
-    if coordinates['sub_category'] == 'bnd':    
-            if ':' in alt:
-                other_coordinates = alt.strip('ACGTN[]').split(':')
-                # BND will often be translocations between different chromosomes
-                other_chrom = other_coordinates[0]
-                coordinates['end_chrom'] = other_coordinates[0].lstrip('chrCHR')
-    
-    coordinates['cytoband_start'] = get_cytoband_coordinates(
-        chrom, position
-    )
-    coordinates['cytoband_end'] = get_cytoband_coordinates(
-        coordinates['end_chrom'], coordinates['end']
-    )
+    
+    sub_category = get_sub_category(alt_len, ref_len, category, svtype)
+    end = get_end(position, alt, category, snvend, svend)
+
+    length = get_length(alt_len, ref_len, category, position, end, svtype, svlen)
+    end_chrom = chrom
+
+    if sub_category == 'bnd':    
+        if ':' in alt:
+            other_coordinates = alt.strip('ACGTN[]').split(':')
+            # BND will often be translocations between different chromosomes
+            other_chrom = other_coordinates[0]
+            if (other_chrom.startswith('chr') or other_chrom.startswith('CHR')):
+                other_chrom = other_chrom[3:]
+            end_chrom = other_chrom
+    
+    cytoband_start = get_cytoband_coordinates(chrom, position)
+    cytoband_end = get_cytoband_coordinates(end_chrom, end)
+
+    coordinates = {
+        'position': position,
+        'end': end,
+        'length': length,
+        'sub_category': sub_category,
+        'mate_id': mate_id,
+        'cytoband_start': cytoband_start,
+        'cytoband_end': cytoband_end,
+        'end_chrom': end_chrom,
+    }
+
 
     return coordinates
diff --git a/scout/parse/variant/variant.py b/scout/parse/variant/variant.py
index 625c8901d..de16616c1 100644
--- a/scout/parse/variant/variant.py
+++ b/scout/parse/variant/variant.py
@@ -81,20 +81,19 @@ def parse_variant(variant, case, variant_type='clinical',
             category = 'snv'
 
     parsed_variant['category'] = category
-    #sub category is 'snv', 'indel', 'del', 'ins', 'dup', 'inv', 'cnv'
-    # 'snv' and 'indel' are subcatogories of snv
-    parsed_variant['sub_category'] = None
 
     ################# General information #################
 
     parsed_variant['reference'] = variant.REF
-    # We allways assume splitted and normalized vcfs
+    
+    ### We allways assume splitted and normalized vcfs!!!
     if len(variant.ALT) > 1:
         raise VcfError("Variants are only allowed to have one alternative")
     parsed_variant['alternative'] = variant.ALT[0]
 
     # cyvcf2 will set QUAL to None if '.' in vcf
     parsed_variant['quality'] = variant.QUAL
+    
     if variant.FILTER:
         parsed_variant['filters'] = variant.FILTER.split(';')
     else:
@@ -109,33 +108,14 @@ def parse_variant(variant, case, variant_type='clinical',
 
     ################# Position specific #################
     parsed_variant['chromosome'] = chrom
-    # position = start
-    parsed_variant['position'] = int(variant.POS)
 
-    svtype = variant.INFO.get('SVTYPE')
-
-    svlen = variant.INFO.get('SVLEN')
-
-    end = int(variant.end)
-
-    mate_id = variant.INFO.get('MATEID')
-
-    coordinates = parse_coordinates(
-        chrom=parsed_variant['chromosome'],
-        ref=parsed_variant['reference'],
-        alt=parsed_variant['alternative'],
-        position=parsed_variant['position'],
-        category=parsed_variant['category'],
-        svtype=svtype,
-        svlen=svlen,
-        end=end,
-        mate_id=mate_id,
-    )
+    coordinates = parse_coordinates(variant, category)
 
+    parsed_variant['position'] = coordinates['position']
     parsed_variant['sub_category'] = coordinates['sub_category']
     parsed_variant['mate_id'] = coordinates['mate_id']
-    parsed_variant['end'] = int(coordinates['end'])
-    parsed_variant['length'] = int(coordinates['length'])
+    parsed_variant['end'] = coordinates['end']
+    parsed_variant['length'] = coordinates['length']
     parsed_variant['end_chrom'] = coordinates['end_chrom']
     parsed_variant['cytoband_start'] = coordinates['cytoband_start']
     parsed_variant['cytoband_end'] = coordinates['cytoband_end']


Test patch:
diff --git a/tests/parse/test_parse_coordinates.py b/tests/parse/test_parse_coordinates.py
new file mode 100644
index 000000000..791148fb3
--- /dev/null
+++ b/tests/parse/test_parse_coordinates.py
@@ -0,0 +1,253 @@
+from scout.parse.variant.coordinates import (get_cytoband_coordinates, get_sub_category, 
+                                             get_length, get_end, parse_coordinates)
+
+
+class CyvcfVariant(object):
+    """Mock a cyvcf variant
+    
+    Default is to return a variant with three individuals high genotype 
+    quality.
+    """
+    def __init__(self, chrom='1', pos=80000, ref='A', alt='C', end=None, 
+                 gt_quals=[60, 60, 60], gt_types=[1, 1, 0], var_type='snv',
+                 info_dict={}):
+        super(CyvcfVariant, self).__init__()
+        self.CHROM = chrom
+        self.POS = pos
+        self.REF = ref
+        self.ALT = [alt]
+        self.end = end or pos
+        self.gt_quals = gt_quals
+        self.gt_types = gt_types
+        self.var_type = var_type
+        self.INFO = info_dict
+
+
+def test_parse_coordinates_snv():
+    variant = CyvcfVariant()
+    
+    coordinates = parse_coordinates(variant, 'snv')
+    
+    assert coordinates['position'] == variant.POS
+
+def test_parse_coordinates_indel():
+    variant = CyvcfVariant(alt='ACCC', end=80003)
+    
+    coordinates = parse_coordinates(variant, 'snv')
+    
+    assert coordinates['position'] == variant.POS
+    assert coordinates['end'] == variant.end
+
+def test_parse_coordinates_translocation():
+    info_dict = {
+        'SVTYPE': 'BND',
+    }
+    variant = CyvcfVariant(
+        ref='N', 
+        alt='N[hs37d5:12060532[',
+        pos=724779,
+        end=724779,
+        var_type='sv',
+        info_dict=info_dict,
+    )
+    
+    coordinates = parse_coordinates(variant, 'sv')
+    
+    assert coordinates['position'] == variant.POS
+    assert coordinates['end'] == 12060532
+    assert coordinates['end_chrom'] == 'hs37d5'
+    assert coordinates['length'] == 10e10
+    assert coordinates['sub_category'] ==  'bnd'
+    
+    
+###### parse subcategory #######
+def test_get_subcategory_snv():
+    alt_len = 1
+    ref_len = 1
+    category = 'snv'
+    svtype = None
+    
+    sub_category = get_sub_category(alt_len, ref_len, category, svtype)
+    
+    assert sub_category == 'snv'
+
+def test_get_subcategory_indel():
+    alt_len = 1
+    ref_len = 3
+    category = 'snv'
+    svtype = None
+    
+    sub_category = get_sub_category(alt_len, ref_len, category, svtype)
+    
+    assert sub_category == 'indel'
+
+###### parse length #######
+
+# get_length(alt_len, ref_len, category, pos, end, svtype=None, svlen=None)
+def test_get_length_snv():
+    alt_len = 1
+    ref_len = 1
+    category = 'snv'
+    pos = end = 879537
+    
+    length = get_length(alt_len, ref_len, category, pos, end)
+    
+    assert length == 1
+
+def test_get_length_indel():
+    alt_len = 3
+    ref_len = 1
+    category = 'snv'
+    pos = end = 879537
+    
+    length = get_length(alt_len, ref_len, category, pos, end)
+    
+    assert length == 2
+
+def test_get_sv_length_small_ins():
+    ## GIVEN an insertion with whole sequence in alt field
+    alt_len = 296
+    ref_len = 1
+    category = 'sv'
+    # Pos and end is same for insertions
+    pos = end = 144343218
+    svtype = 'ins'
+    svlen = 296
+    
+    ## WHEN parsing the length
+    length = get_length(alt_len, ref_len, category, pos, end, svtype, svlen)
+    
+    ## THEN assert that the length is correct
+    assert length == 296
+
+def test_get_sv_length_large_ins_no_length():
+    ## GIVEN an imprecise insertion
+    alt_len = 5
+    ref_len = 1
+    category = 'sv'
+    # Pos and end is same for insertions
+    pos = end = 133920667
+    svtype = 'ins'
+    svlen = None
+    
+    ## WHEN parsing the length
+    length = get_length(alt_len, ref_len, category, pos, end, svtype, svlen)
+    
+    ## THEN assert that the length is correct
+    assert length == -1
+
+def test_get_sv_length_translocation():
+    ## GIVEN an translocation
+    alt_len = 16
+    ref_len = 1
+    category = 'sv'
+    pos = 726044
+    end = None
+    svtype = 'bnd'
+    svlen = None
+    
+    ## WHEN parsing the length
+    length = get_length(alt_len, ref_len, category, pos, end, svtype, svlen)
+    
+    ## THEN assert that the length is correct
+    assert length == 10e10
+
+def test_get_sv_length_cnvnator_del():
+    ## GIVEN an cnvnator type deletion
+    alt_len = 5
+    ref_len = 1
+    category = 'sv'
+    pos = 1
+    end = 10000
+    svtype = 'del'
+    svlen = -10000
+    
+    ## WHEN parsing the length
+    length = get_length(alt_len, ref_len, category, pos, end, svtype, svlen)
+    
+    ## THEN assert that the length is correct
+    assert length == 10000
+
+def test_get_sv_length_del_no_length():
+    ## GIVEN an deletion without len
+    alt_len = 5
+    ref_len = 1
+    category = 'sv'
+    pos = 869314
+    end = 870246
+    svtype = 'del'
+    svlen = None
+    
+    ## WHEN parsing the length
+    length = get_length(alt_len, ref_len, category, pos, end, svtype, svlen)
+    
+    ## THEN assert that the length is correct
+    assert length == end - pos
+
+###### parse end #######
+# get_end(pos, alt, category, snvend, svend, svlen)
+
+# snv/indels are easy since cyvcf2 are parsing the end for us
+
+def test_get_end_snv():
+    alt = 'T'
+    category = 'snv'
+    pos = snvend = 879537
+    
+    end = get_end(pos, alt, category, snvend, svend=None, svlen=None)
+    
+    assert end == snvend
+
+def test_get_end_indel():
+    alt = 'C'
+    category = 'indel'
+    pos = 302253
+    snvend = 302265
+    
+    end = get_end(pos, alt, category, snvend, svend=None, svlen=None)
+    
+    assert end == snvend
+
+# SVs are much harder since there are a lot of corner cases
+# Most SVs (except translocations) have END annotated in INFO field
+# The problem is that many times END==POS and then we have to do some magic on our own
+
+def test_get_end_tiddit_translocation():
+    ## GIVEN a translocation
+    alt = 'N[hs37d5:12060532['
+    category = 'sv'
+    pos = 724779
+    
+    ## WHEN parsing the end coordinate
+    end = get_end(pos, alt, category, snvend=None, svend=None, svlen=None)
+    
+    ## THEN assert that the end is the same as en coordinate described in alt field
+    assert end == 12060532
+
+def test_get_end_tiddit_translocation():
+    ## GIVEN a translocation
+    alt = 'N[hs37d5:12060532['
+    category = 'sv'
+    pos = 724779
+    
+    ## WHEN parsing the end coordinate
+    end = get_end(pos, alt, category, snvend=None, svend=None, svlen=None)
+    
+    ## THEN assert that the end is the same as en coordinate described in alt field
+    assert end == 12060532
+
+def test_get_end_deletion():
+    ## GIVEN a translocation
+    alt = '<DEL>'
+    category = 'sv'
+    pos = 869314
+    svend = 870246
+    svlen = None
+    
+    ## WHEN parsing the end coordinate
+    end = get_end(pos, alt, category, snvend=None, svend=svend, svlen=svlen)
+    
+    ## THEN assert that the end is the same as en coordinate described in alt field
+    assert end == svend
+
+