Repository: LMFDB/lmfdb

Issue:
Proxy error for some higher genus w/autos download
(possibly related to #1079)

If I go here:
http://beta.lmfdb.org/HigherGenus/C/Aut/10.6-1.0.2-2-2-2-2-2-2-2-2-2.1
and click on the "Download Magma Code" or "Download Gap Code" buttons, I get a proxy error. I suspect the issue is that the download is creating a file with about 3280 generating vectors (and each vector creates about 10 lines of text in the download file).  I can modify the code to not allow downloads quite that big, but I wasn't sure what the cutoff should be, i.e. what "too many generating vectors" means.

A page like this:
http://beta.lmfdb.org/HigherGenus/C/Aut/10.10-1.0.2-2-2-2-2-2-5
downloads just fine (and that would be 1250 generating vectors between the two passports). 



Discussion:
@jenpaulhus I can fix this for you (3280*10 lines is not that big and should be fine), your download code just needs a little tweaking to make it faster; I'll submit a PR for this shortly.


Resolving patch:
diff --git a/lmfdb/higher_genus_w_automorphisms/main.py b/lmfdb/higher_genus_w_automorphisms/main.py
--- a/lmfdb/higher_genus_w_automorphisms/main.py
+++ b/lmfdb/higher_genus_w_automorphisms/main.py
@@ -11,8 +11,7 @@
 from lmfdb import base
 from flask import render_template, request, url_for, make_response, redirect
 from lmfdb.utils import to_dict, random_value_from_collection, flash_error
-from lmfdb.search_parsing import search_parser, parse_ints, parse_count, parse_start, clean_input
-BRACKETED_POSINT_RE = re.compile(r'^\[\]|\[\d+(,\d+)*\]$')
+from lmfdb.search_parsing import parse_ints, parse_count, parse_start, clean_input, parse_bracketed_posints, parse_gap_id
 
 from sage.all import Permutation
 from lmfdb.higher_genus_w_automorphisms import higher_genus_w_automorphisms_page
@@ -107,56 +106,6 @@ def label_to_breadcrumbs(L):
     newsig += ']'    
     return newsig
 
-
-
-#copied from parse_bracketed_posints,  but keeps outside brackets in string
-@search_parser(clean_info=True) # see SearchParser.__call__ for actual arguments when calling
-def parse_bracketed_posints2(inp, query, qfield, maxlength=None, exactlength=None, split=True, process=None, check_divisibility=None):
-    if process is None: process = lambda x: x
-    if (not BRACKETED_POSINT_RE.match(inp) or
-        (maxlength is not None and inp.count(',') > maxlength - 1) or
-        (exactlength is not None and inp.count(',') != exactlength - 1) or
-        (exactlength is not None and inp == '[]' and exactlength > 0)):
-        if exactlength == 2:
-            lstr = "pair of integers"
-            example = "[2,3] or [3,3]"
-        elif exactlength == 1:
-            lstr = "list of 1 integer"
-            example = "[2]"
-        elif exactlength is not None:
-            lstr = "list of %s integers" % exactlength
-            example = str(range(2,exactlength+2)).replace(" ","") + " or " + str([3]*exactlength).replace(" ","")
-        elif maxlength is not None:
-            lstr = "list of at most %s integers" % maxlength
-            example = str(range(2,maxlength+2)).replace(" ","") + " or " + str([2]*max(1, maxlength-2)).replace(" ","")
-        else:
-            lstr = "list of integers"
-            example = "[1,2,3] or [5,6]"
-        raise ValueError("It needs to be a %s in square brackets, such as %s." % (lstr, example))
-    else:
-        if inp == '[]': # fixes bug in the code below (split never returns an empty list)
-            query[qfield] = []
-            return
-        if check_divisibility == 'decreasing':
-            # Check that each entry divides the previous
-            L = [int(a) for a in inp[1:-1].split(',')]
-            for i in range(len(L)-1):
-                if L[i] % L[i+1] != 0:
-                    raise ValueError("Each entry must divide the previous, such as [4,2].")
-        elif check_divisibility == 'increasing':
-            # Check that each entry divides the previous
-            L = [int(a) for a in inp[1:-1].split(',')]
-            for i in range(len(L)-1):
-                if L[i+1] % L[i] != 0:
-                    raise ValueError("Each entry must divide the next, such as [2,4].")
-        if split:
-#            query[qfield] = [process(int(a)) for a in inp[1:-1].split(',')]
-            query[qfield] = [process(int(a)) for a in inp.split(',')]
-        else:
-#            query[qfield] = inp[1:-1]
-            query[qfield] = inp
-
-    
 @higher_genus_w_automorphisms_page.route("/")
 def index():
     bread = get_bread()
@@ -175,8 +124,6 @@ def index():
     return render_template("hgcwa-index.html", title="Families of Higher Genus Curves with Automorphisms", bread=bread, info=info, learnmore=learnmore)
 
 
-
-
 @higher_genus_w_automorphisms_page.route("/random")
 def random_passport():
     C = base.getDBConnection()
@@ -225,9 +172,9 @@ def higher_genus_w_automorphisms_search(**args):
         info.update({'signature': str(sig)})
             
     try:
-        parse_bracketed_posints2(info,query,'group', split=False, exactlength=2, name='Group')
+        parse_gap_id(info,query,'group','Group')
         parse_ints(info,query,'genus',name='Genus')
-        parse_bracketed_posints2(info,query,'signature',split=False,name='Signature')
+        parse_bracketed_posints(info,query,'signature',split=False,name='Signature',keepbrackets=True)
         parse_ints(info,query,'dim',name='Dimension of the family')
         if 'inc_hyper' in info:
             if info['inc_hyper'] == 'exclude':
@@ -235,7 +182,7 @@ def higher_genus_w_automorphisms_search(**args):
             elif info['inc_hyper'] == 'only':
                 query['hyperelliptic'] = True
 
-        query['cc.1'] = 1       
+        query['cc.1'] = 1
 
     except ValueError:
         return search_input_error(info, bread)
@@ -546,6 +493,7 @@ def hgcwa_code_download(**args):
 Comment = {'magma': '//', 'gap': '#'}
 
 def hgcwa_code(**args):
+    import time
     label = args['label']
     C = base.getDBConnection()
     lang = args['download_type']
@@ -561,8 +509,6 @@ def hgcwa_code(**args):
     elif label_is_one_family(label):
         data = C.curve_automorphisms.passports.find({"label" : label})
 
-
-
     code += Comment[lang] + code_list['gp_comment'][lang] +'\n'
     code += code_list['group'][lang] + str(data[0]['group'])+ ';\n'
 
@@ -577,37 +523,36 @@ def hgcwa_code(**args):
 
     code += '\n'
 
-    for dataz in data:
-        code += Comment[lang] + " Here we add an action to result_record." + '\n'
-        for k in depends_on_action:
-            code += code_list[k][lang] + str(dataz[k])+ ';\n'
-
-        if lang == 'magma':
-            code += code_list['con'][lang] + str(dataz['con'])+ ';\n' 
-             
-        code += code_list['gen_gp'][lang]+ '\n'
-        code += code_list['passport_label'][lang] + str(dataz['cc'][0]) + ';\n'
-        code += code_list['gen_vect_label'][lang] + str(dataz['cc'][1]) + ';\n'
-        
-#cannot have full auto + hyperelliptic in data
-        if 'signH' in dataz:
-            code += code_list['full_auto'][lang]+str(dataz['full_auto']) + ';\n'
-            code += code_list['full_sign'][lang]+str(dataz['signH']) + ';\n'        
-            code+=code_list['add_to_total_full'][lang]+'\n'
-
-        if 'hyperelliptic' in dataz:            
-                                               
-            if dataz['hyperelliptic']:
-                code +=code_list['hyp'][lang]+ code_list['tr'][lang] + ';\n'
-                code += code_list['hyp_inv'][lang]+str(dataz['hyp_involution']) + code_list['hyp_inv_last'][lang]
-                code +=code_list['add_to_total_hyp'][lang]+'\n'
-
-            else:
-                code +=code_list['hyp'][lang]+ code_list['fal'][lang] + ';\n'
-                code +=code_list['add_to_total_basic'][lang]+'\n'
-
-        code += '\n'
+    # create formatting templates to be filled in with each record in data
+    startstr = Comment[lang] + ' Here we add an action to result_record.\n'
+    stdfmt = ''
+    for k in depends_on_action:
+        stdfmt += code_list[k][lang] + '{' + k + '}'+ ';\n'
 
+    if lang == 'magma':
+        stdfmt += code_list['con'][lang] + '{con}' + ';\n' 
+         
+    stdfmt += code_list['gen_gp'][lang]+ '\n'
+    stdfmt += code_list['passport_label'][lang] + '{cc[0]}' + ';\n'
+    stdfmt += code_list['gen_vect_label'][lang] + '{cc[1]}' + ';\n'
+    
+    # extended formatting template for when signH is present
+    signHfmt = stdfmt
+    signHfmt = code_list['full_auto'][lang] + '{full_auto}' + ';\n'
+    signHfmt += code_list['full_sign'][lang] + '{signH}' + ';\n'        
+    signHfmt += code_list['add_to_total_full'][lang] + '\n'
+
+    # additional info for hyperelliptic cases
+    hypfmt = code_list['hyp'][lang] + code_list['tr'][lang] + ';\n'
+    hypfmt += code_list['hyp_inv'][lang] + '{hyp_involution}' + code_list['hyp_inv_last'][lang]
+    hypfmt += code_list['add_to_total_hyp'][lang] + '\n'
+    nhypstr = code_list['hyp'][lang] + code_list['fal'][lang] + ';\n'
+    nhypstr += code_list['add_to_total_basic'][lang] + '\n'
+
+    start = time.time()
+    lines = [(startstr + (signHfmt if 'signH' in dataz else stdfmt).format(**dataz) + ((hypfmt.format(**dataz) if dataz['hyperelliptic'] else nhypstr) if 'hyperelliptic' in dataz else '')) for dataz in data]
+    code += '\n'.join(lines)
+    print "%s seconds for %d bytes" %(time.time() - start,len(code))
     return code
 
 
diff --git a/lmfdb/search_parsing.py b/lmfdb/search_parsing.py
--- a/lmfdb/search_parsing.py
+++ b/lmfdb/search_parsing.py
@@ -312,7 +312,7 @@ def parse_primes(inp, query, qfield, mode=None, to_string=False):
         raise ValueError("It needs to be a prime (such as 5), or a comma-separated list of primes (such as 2,3,11).")
 
 @search_parser(clean_info=True) # see SearchParser.__call__ for actual arguments when calling
-def parse_bracketed_posints(inp, query, qfield, maxlength=None, exactlength=None, split=True, process=None, check_divisibility=None):
+def parse_bracketed_posints(inp, query, qfield, maxlength=None, exactlength=None, split=True, process=None, check_divisibility=None, keepbrackets=False):
     if process is None: process = lambda x: x
     if (not BRACKETED_POSINT_RE.match(inp) or
         (maxlength is not None and inp.count(',') > maxlength - 1) or
@@ -353,7 +353,10 @@ def parse_bracketed_posints(inp, query, qfield, maxlength=None, exactlength=None
         if split:
             query[qfield] = [process(int(a)) for a in inp[1:-1].split(',')]
         else:
-            query[qfield] = inp[1:-1]
+            query[qfield] = inp if keepbrackets else inp[1:-1]
+
+def parse_gap_id(info, query, field='group', name='group', qfield='group'):
+    parse_bracketed_posints(info,query,'group', split=False, exactlength=2, keepbrackets=True, name='Group')
 
 @search_parser(clean_info=True, default_field='galois_group', default_name='Galois group', default_qfield='galois') # see SearchParser.__call__ for actual arguments when calling
 def parse_galgrp(inp, query, qfield, use_bson=True):


Test patch: