Repository: LMFDB/lmfdb

Issue:
Weil polynomial issues
From the feedback page from Stefano Marseglia (stefanom@math.su.se):

I think there is a problem with the search function when setting the initial coefficient to 0 (and eventually something else). It seems that the 0's are ignored.

I believe that the Weil polynomials with real roots are missing from the database.
For example the polynomial (p*x^2-1) does not correspond to an isogeny class of elliptic curves over F_p, but it's square defines an isogeny class of surfaces (over F_p), see (1971, Milne, Waterhouse - Abelian Varieties over Finite Fields, Theorem 8).
I think that polynomials of this form and their multiples are missing.



Discussion:
For p = 2, I'm not seeing (1-2x^2)^2 in dimension 2 nor (1-2x^2)^4 in dimension 4.

I just checked my original code that generates Weil polynomials and it does pick up this polynomial; so the issue is probably with the code that filters for the Milne-Waterhouse condition.

Resolving patch:
diff --git a/lmfdb/abvar/fq/main.py b/lmfdb/abvar/fq/main.py
--- a/lmfdb/abvar/fq/main.py
+++ b/lmfdb/abvar/fq/main.py
@@ -4,6 +4,7 @@
 
 from flask import flash, render_template, url_for, request, redirect, send_file
 from markupsafe import Markup
+from collections import defaultdict
 from sage.rings.all import PolynomialRing, ZZ
 
 from lmfdb import db
@@ -40,6 +41,7 @@ def ECFq_redirect():
 def learnmore_list():
     return [('Completeness of the data', url_for(".completeness_page")),
             ('Source of the data', url_for(".how_computed_page")),
+            ('Reliability of the data', url_for(".reliability_page")),
             ('Labels', url_for(".labels_page"))]
 
 # Return the learnmore list with the matchstring entry removed
@@ -230,7 +232,7 @@ def abelian_variety_browse(**args):
         else:
             qmin = table_params['q'].get('$gte',min(qs) if qs else table_params['q'].get('$lte',0))
             qmax = table_params['q'].get('$lte',max(qs) if qs else table_params['q'].get('$gte',1000))
-    info['table'] = {}
+    info['table'] = defaultdict(lambda: defaultdict(int))
     if gmin == gmax:
         info['table_dimension_range'] = "{0}".format(gmin)
     else:
@@ -240,16 +242,9 @@ def abelian_variety_browse(**args):
     else:
         info['table_field_range'] = "{0}-{1}".format(qmin, qmax)
 
-    for q in qs:
-        if q < qmin or q > qmax:
-            continue
-        info['table'][q] = {}
-        L = av_stats._counts[q]
-        for g in xrange(gmin, gmax+1):
-            if g < len(L):
-                info['table'][q][g] = L[g]
-            else:
-                info['table'][q][g] = 0
+    for (g,q), cnt in av_stats._counts.items():
+        if qmin <= q <= qmax and gmin <= g <= gmax:
+            info['table'][q][g] = cnt
 
     info['col_heads'] = [q for q in qs if q >= qmin and q <= qmax]
     info['row_heads'] = [g for g in gs if g >= gmin and g <= gmax]
@@ -285,6 +280,13 @@ def completeness_page():
     return render_template("single.html", kid='dq.av.fq.extent',
                            credit=abvarfq_credit, title=t, bread=bread, learnmore=learnmore_list_remove('Completeness'))
 
+@abvarfq_page.route("/Reliability")
+def reliability_page():
+    t = 'Reliability of the Weil Polynomial Data'
+    bread = get_bread(('Reliability', '.'))
+    return render_template("single.html", kid='dq.av.fq.reliability',
+                           credit=abvarfq_credit, title=t, bread=bread, learnmore=learnmore_list_remove('Reliability'))
+
 @abvarfq_page.route("/Source")
 def how_computed_page():
     t = 'Source of the Weil Polynomial Data'
diff --git a/lmfdb/abvar/fq/stats.py b/lmfdb/abvar/fq/stats.py
--- a/lmfdb/abvar/fq/stats.py
+++ b/lmfdb/abvar/fq/stats.py
@@ -1,32 +1,24 @@
 # -*- coding: utf-8 -*-
+from collections import defaultdict
 from lmfdb import db
-from lmfdb.utils import comma
+from lmfdb.utils import comma, StatsDisplay
 from lmfdb.logger import make_logger
-from sage.structure.unique_representation import UniqueRepresentation
 from sage.misc.lazy_attribute import lazy_attribute
 
 logger = make_logger("abvarfq")
 
-class AbvarFqStats(UniqueRepresentation):
-    def __init__(self):
-        logger.debug("Constructing AbvarFqStats")
-
+class AbvarFqStats(StatsDisplay):
     @lazy_attribute
     def _counts(self):
-        logger.debug("Looking up abelian variety counts")
-        D = {}
-        for q, L in db.av_fqisog.stats.get_oldstat('counts').iteritems():
-            D[int(q)] = L
-        return D
+        return db.av_fqisog.stats.column_counts(['g', 'q'])
 
     @lazy_attribute
     def qs(self):
-        return sorted(self._counts.keys())
+        return sorted(set(q for g,q in self._counts))
 
     @lazy_attribute
     def gs(self):
-        maxg = max(len(L)-1 for L in self._counts.values())
-        return range(1, maxg+1)
+        return sorted(set(g for g,q in self._counts))
 
     @lazy_attribute
     def counts(self):
@@ -34,25 +26,19 @@ def counts(self):
         counts['nclasses'] = ncurves = sum(sum(L) for L in self._counts.itervalues())
         counts['nclasses_c'] = comma(ncurves)
         counts['gs'] = self.gs
-        counts['qs'] = qs = self.qs
-        counts['qg_count'] = {}
-        for q in qs:
-            counts['qg_count'][q] = {}
-            L = self._counts[q]
-            for g in xrange(1,self.maxg[None]+1):
-                if g < len(L):
-                    counts['qg_count'][q][g] = L[g]
-                else:
-                    counts['qg_count'][q][g] = 0
+        counts['qs'] = self.qs
+        counts['qg_count'] = defaultdict(lambda: defaultdict(int))
+        for (g,q), cnt in self._counts.items():
+            counts['qg_count'][q][g] = cnt
         return counts
 
     @lazy_attribute
     def maxq(self):
-        return {g: max(q for q in self.qs if len(self._counts[q]) > g) for g in self.gs}
+        return {g: max(q for gg,q in self._counts if g==gg) for g in self.gs}
 
     @lazy_attribute
     def maxg(self):
-        maxg = {q: len(self._counts[q]) - 1 for q in self.qs}
+        maxg = {q: max(g for g,qq in self._counts if q==qq) for q in self.qs}
         # maxg[None] used in decomposition search
         maxg[None] = max(self.gs)
         return maxg


Test patch:
diff --git a/lmfdb/abvar/fq/test_browse_page.py b/lmfdb/abvar/fq/test_browse_page.py
--- a/lmfdb/abvar/fq/test_browse_page.py
+++ b/lmfdb/abvar/fq/test_browse_page.py
@@ -53,8 +53,8 @@ def test_table_range(self):
         r"""
         Check that we can change the table range
         """
-        self.check_args("/Variety/Abelian/Fq/?table_field_range=32-100&table_dimension_range=2-4",'6408')
-        self.check_args("/Variety/Abelian/Fq/?table_field_range=4..7&table_dimension_range=2..4",'2944')
+        self.check_args("/Variety/Abelian/Fq/?table_field_range=32-100&table_dimension_range=2-4",'6409')
+        self.check_args("/Variety/Abelian/Fq/?table_field_range=4..7&table_dimension_range=2..4",'2953')
         # and that it deals with invalid input
         self.check_args("/Variety/Abelian/Fq/?table_field_range=2-27&table_dimension_range=1%2C3%2C5",'Error: You cannot use commas in the table ranges.')
         self.check_args("/Variety/Abelian/Fq/?table_field_range=2-27&table_dimension_range=x",'not a valid input')
@@ -187,7 +187,7 @@ def test_search_jacobian(self):
         self.not_check_args("/Variety/Abelian/Fq/?q=3&g=2&jacobian=no",'2.3.ae_i')
         self.not_check_args("/Variety/Abelian/Fq/?q=3&g=2&jacobian=no",'2.3.ae_i')
         # unknowns
-        self.check_args("/Variety/Abelian/Fq/?g=3&jacobian=not_yes&polarizable=yes",'No matches')
+        self.check_args("/Variety/Abelian/Fq/?g=3&jacobian=not_yes&polarizable=yes",'3.3.a_ad_a')
         self.check_args("/Variety/Abelian/Fq/?q=2&g=3&p_rank=0&jacobian=not_no",'3.2.c_c_c')
 
     def test_search_princpol(self):
@@ -251,4 +251,4 @@ def test_search_combos(self):
         self.check_args("/Variety/Abelian/Fq/?initial_coefficients=%5B1%2C-1%2C3%2C9%5D&abvar_point_count=%5B75%2C7125%5D", 'No matches')
         # Combining unknown fields on Jacobian and Principal polarization.
         self.check_args("/Variety/Abelian/Fq/?g=3&jacobian=no&polarizable=not_no", '3.2.a_a_ae')
-        self.check_args("/Variety/Abelian/Fq/?g=3&jacobian=no&polarizable=yes", 'No matches')
+        self.check_args("/Variety/Abelian/Fq/?g=3&jacobian=no&polarizable=yes", '3.2.a_ac_a')