Repository: LMFDB/lmfdb

Issue:
Page Not Found redirects to beta
It is possible this is working as intended, but if I go to www.lmfdb.org and try to go to a link that doesn't exist like www.lmfdb.org/junk,  it redirects me to beta to tell me that the page was not found.  This doesn't seem to happen if I type a nonsense urls for the individual sections (like EC or HGCWA).




Discussion:
This is working as designed (whether that design is optimal is a separate question, I'm just noting that this is not a bug).  Do you think it should behave differently?
It was a little strange that the sidebar changed. I'm not sure if it's better or worse, just unexpected for me. It made me wonder whether I had typed the 2nd part of the URL wrong  or if I had accidentally been in beta from the start.
The tricky part here is that we want to redirect routes like https://www.lmfdb.org/Motive/Hypergeometric/Q/ which are not yet in production to beta, but it is a pain (and high maintenance) to keep track of all the routes that are not valid in production but might be valid on beta, so we just redirect anything that we know is not valid on production.

I agree that it would be a better user-experience to just throw a 404 on URLs like https://www.lmfdb.org/junk, and if there is an easy way to do that, I would be in favor.  Do you have any thoughts on this @edgarcosta?
I discussed this issue with @edgarcosta and I think he has a plan to address it, is that correct @edgarcosta ?

Resolving patch:
diff --git a/lmfdb/app.py b/lmfdb/app.py
--- a/lmfdb/app.py
+++ b/lmfdb/app.py
@@ -1,15 +1,26 @@
-
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+from .utils.config import get_secret_key
 import os
 from socket import gethostname
 import time
 import six
 from urllib.parse import urlparse, urlunparse
 
-from flask import (Flask, g, render_template, request, make_response,
-                   redirect, url_for, current_app, abort)
+from flask import (
+    Flask,
+    abort,
+    current_app,
+    g,
+    make_response,
+    redirect,
+    render_template,
+    request,
+    url_for,
+)
+from markupsafe import escape
 from sage.env import SAGE_VERSION
+from sage.all import cached_function
 # acknowledgment page, reads info from CONTRIBUTORS.yaml
 
 from .logger import logger_file_handler, critical
@@ -21,6 +32,7 @@
 #         Main app         #
 ############################
 
+
 class ReverseProxied(object):
     def __init__(self, app):
         self.app = app
@@ -31,6 +43,7 @@ def __call__(self, environ, start_response):
             environ['wsgi.url_scheme'] = scheme
         return self.app(environ, start_response)
 
+
 app = Flask(__name__)
 
 app.wsgi_app = ReverseProxied(app.wsgi_app)
@@ -39,22 +52,30 @@ def __call__(self, environ, start_response):
 # App attribute functions  #
 ############################
 
+
 def is_debug_mode():
-    from flask import current_app
     return current_app.debug
 
 # this is set here and is available for ctx_proc_userdata
+
+
 @app.before_request
 def set_beta_state():
-    g.BETA = (os.getenv('BETA')=='1') or is_debug_mode()
+    g.BETA = (os.getenv('BETA') == '1') or is_debug_mode()
+
 
 def is_beta():
     from flask import g
     return g.BETA
 
+
 app.is_running = False
+
+
 def set_running():
     app.is_running = True
+
+
 def is_running():
     return app.is_running
 
@@ -62,6 +83,7 @@ def is_running():
 # Global app configuration #
 ############################
 
+
 app.logger.addHandler(logger_file_handler())
 
 # If the debug toolbar is installed then use it
@@ -74,7 +96,6 @@ def is_running():
 
 # secret key, necessary for sessions, and sessions are
 # in turn necessary for users to login
-from .utils.config import get_secret_key
 app.secret_key = get_secret_key()
 
 # tell jinja to remove linebreaks
@@ -91,6 +112,8 @@ def is_running():
 #  * title = 'LMFDB'
 #  * meta_description, shortthanks, feedbackpage
 #  * DEBUG and BETA variables storing whether running in each mode
+
+
 @app.context_processor
 def ctx_proc_userdata():
     # insert an empty info={} as default
@@ -134,6 +157,8 @@ def modify_url(**replace):
 #app.jinja_env.globals['sidebar'] = get_sidebar()
 #
 # so instead we do this to ensure that the sidebar content is available to every page:
+
+
 @app.context_processor
 def inject_sidebar():
     from .homepage import get_sidebar
@@ -143,21 +168,23 @@ def inject_sidebar():
 # Bottom link to google code #
 ##############################
 
+
 branch = "web"
-if (os.getenv('BETA')=='1'):
+if (os.getenv('BETA') == '1'):
     branch = "dev"
 
+
 def git_infos():
     try:
         from subprocess import Popen, PIPE
         # cwd should be the root of git repo
-        cwd = os.path.join(os.path.dirname(os.path.realpath(__file__)),"..")
+        cwd = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
         commands = ['''git rev-parse HEAD''',
                     '''git show --format="%ci" -s HEAD''',
                     '''git branch --contains HEAD''',
                     '''git reflog -n5''',
                     '''git log --graph  -n 10''']
-        kwdargs = {'shell': True, 'stdout' : PIPE, 'cwd' : cwd}
+        kwdargs = {'shell': True, 'stdout': PIPE, 'cwd': cwd}
         if six.PY3:
             kwdargs['encoding'] = 'utf-8'
         pairs = [(c, Popen(c, **kwdargs).communicate()[0]) for c in commands]
@@ -169,7 +196,7 @@ def git_infos():
         return '-', '-', '-'
 
 
-git_rev, git_date, _  = git_infos()
+git_rev, git_date, _ = git_infos()
 
 # Creates link to the source code at the most recent commit.
 _url_source = 'https://github.com/LMFDB/lmfdb/tree/'
@@ -179,6 +206,7 @@ def git_infos():
 _url_changeset = 'https://github.com/LMFDB/lmfdb/commits/%s' % branch
 _latest_changeset = '<a href="%s">%s</a>' % (_url_changeset, git_date)
 
+
 @app.context_processor
 def link_to_current_source():
     return {'current_source': _current_source,
@@ -193,6 +221,8 @@ def link_to_current_source():
 # {{ <datetimeobject> | fmtdatetime }} you can format it inside a jinja template
 # if you want to do more than just the default, use it for example this way:
 # {{ <datetimeobject>|fmtdatetime('%H:%M:%S') }}
+
+
 @app.template_filter("fmtdatetime")
 def fmtdatetime(value, format='%Y-%m-%d %H:%M:%S'):
     import datetime
@@ -202,11 +232,15 @@ def fmtdatetime(value, format='%Y-%m-%d %H:%M:%S'):
         return "-"
 
 # You can use this formatter to turn newlines in a string into HTML line breaks
+
+
 @app.template_filter("nl2br")
 def nl2br(s):
     return s.replace('\n', '<br/>\n')
 
 # You can use this formatter to encode a dictionary into a url string
+
+
 @app.template_filter('urlencode')
 def urlencode(kwargs):
     from six.moves.urllib.parse import urlencode
@@ -246,26 +280,29 @@ def netloc_redirect():
         urlparts.netloc == "www.lmfdb.org"
         and
         not white_listed(urlparts.path)
+        and valid_bread(urlparts.path)
     ):
         replaced = urlparts._replace(netloc="beta.lmfdb.org", scheme="https")
         return redirect(urlunparse(replaced), code=302)
 
 
-
 def timestamp():
     return '[%s UTC]' % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
 
+
 @app.errorhandler(404)
 def not_found_404(error):
-    app.logger.info('%s 404 error for URL %s %s'%(timestamp(),request.url,error.description))
-    messages = error.description if isinstance(error.description,(list,tuple)) else (error.description,)
+    app.logger.info('%s 404 error for URL %s %s' % (timestamp(), request.url, error.description))
+    messages = error.description if isinstance(error.description, (list, tuple)) else (error.description,)
     return render_template("404.html", title='LMFDB Page Not Found', messages=messages), 404
 
+
 @app.errorhandler(500)
 def not_found_500(error):
-    app.logger.error("%s 500 error on URL %s %s"%(timestamp(),request.url, error.args))
+    app.logger.error("%s 500 error on URL %s %s" % (timestamp(), request.url, error.args))
     return render_template("500.html", title='LMFDB Error'), 500
 
+
 @app.errorhandler(503)
 def not_found_503(error):
     return render_template("503.html"), 503
@@ -274,6 +311,7 @@ def not_found_503(error):
 #           Cookies          #
 ##############################
 
+
 @app.before_request
 def get_menu_cookie():
     """
@@ -285,18 +323,21 @@ def get_menu_cookie():
 #       Top-level pages      #
 ##############################
 
+
 @app.route("/")
 def index():
     return render_template('index-boxes.html',
-        titletag="The L-functions and modular forms database",
-        title="The L-functions and modular forms database (LMFDB)",
-        bread=None,
-        boxes=load_boxes())
+                           titletag="The L-functions and modular forms database",
+                           title="The L-functions and modular forms database (LMFDB)",
+                           bread=None,
+                           boxes=load_boxes())
+
 
 @app.route("/about")
 def about():
     return render_template("about.html", title="About the LMFDB")
 
+
 @app.route("/health")
 @app.route("/alive")
 def alive():
@@ -310,7 +351,6 @@ def alive():
         abort(503)
 
 
-
 @app.route("/statshealth")
 def statshealth():
     """
@@ -341,6 +381,7 @@ def statshealth():
     else:
         abort(503)
 
+
 @app.route("/info")
 def info():
     output = ""
@@ -351,7 +392,7 @@ def info():
         output += "db is offline\n"
     else:
         conn_str = "%s" % db.conn
-        output += "Connection: %s\n" % conn_str.replace("<","").replace(">","")
+        output += "Connection: %s\n" % conn_str.replace("<", "").replace(">", "")
         output += "User: %s\n" % db._user
         output += "Read only: %s\n" % db._read_only
         output += "Read and write to userdb: %s\n" % db._read_and_write_userdb
@@ -364,115 +405,138 @@ def info():
 
 @app.route("/acknowledgment")
 def acknowledgment():
-    bread = [("Acknowledgments" , '')]
+    bread = [("Acknowledgments", '')]
     return render_template("acknowledgment.html", title="Acknowledgments", contribs=contribs, bread=bread)
 
+
 @app.route("/acknowledgment/activities")
 def workshops():
-    bread = [("Acknowledgments" , url_for('.acknowledgment')) , ("Activities", '')]
+    bread = [("Acknowledgments", url_for('.acknowledgment')), ("Activities", '')]
     return render_template("workshops.html", title="LMFDB Activities", contribs=contribs, bread=bread)
 
+
 @app.route("/lucant")
 @app.route("/LuCaNT")
 def lucant():
-    bread = [("LuCaNT" , '')]
+    bread = [("LuCaNT", '')]
     return render_template("lucant.html", title="LMFDB, Computation, and Number Theory (LuCaNT)", contribs=contribs, bread=bread)
 
 # google's CSE for www.lmfdb.org/* (and *only* those pages!)
+
+
 @app.route("/search")
 def search():
     return render_template("search.html", title="Search LMFDB", bread=[('Search', url_for("search"))])
 
+
 @app.route('/ModularForm')
 @app.route('/ModularForm/')
 def modular_forms():
     t = 'Modular forms'
     b = [(t, url_for('modular_forms'))]
     # lm = [('History of modular forms', '/ModularForm/history')]
-    return render_template('single.html', title=t, kid='mf.about', bread=b) #, learnmore=lm)
+    return render_template('single.html', title=t, kid='mf.about', bread=b)  # , learnmore=lm)
 
 # @app.route("/ModularForm/history")
+
+
 def modular_forms_history():
     t = 'Modular forms'
     b = [(t, url_for('modular_forms'))]
     b.append(('History', url_for("modular_forms_history")))
     return render_template(_single_knowl, title="A brief history of modular forms", kid='mf.gl2.history', body_class=_bc, bread=b)
 
+
 @app.route('/Variety')
 @app.route('/Variety/')
 def varieties():
     t = 'Varieties'
     b = [(t, url_for('varieties'))]
     # lm = [('History of varieties', '/Variety/history')]
-    return render_template('single.html', title=t, kid='varieties.about', bread=b) #, learnmore=lm)
+    return render_template('single.html', title=t, kid='varieties.about', bread=b)  # , learnmore=lm)
 
 # @app.route("/Variety/history")
+
+
 def varieties_history():
     t = 'Varieties'
     b = [(t, url_for('varieties'))]
     b.append(('History', url_for("varieties_history")))
     return render_template(_single_knowl, title="A brief history of varieties", kid='ag.variety.history', body_class=_bc, bread=b)
 
+
 @app.route('/Field')
 @app.route('/Field/')
 def fields():
     t = 'Fields'
     b = [(t, url_for('fields'))]
     # lm = [('History of fields', '/Field/history')]
-    return render_template('single.html', kid='field.about', title=t, body_class=_bc, bread=b) #, learnmore=lm)
+    return render_template('single.html', kid='field.about', title=t, body_class=_bc, bread=b)  # , learnmore=lm)
 
 # @app.route("/Field/history")
+
+
 def fields_history():
     t = 'Fields'
     b = [(t, url_for('fields'))]
     b.append(('History', url_for("fields_history")))
     return render_template(_single_knowl, title="A brief history of fields", kid='field.history', body_class=_bc, bread=b)
 
+
 @app.route('/Representation')
 @app.route('/Representation/')
 def representations():
     t = 'Representations'
     b = [(t, url_for('representations'))]
     # lm = [('History of representations', '/Representation/history')]
-    return render_template('single.html', kid='repn.about', title=t, body_class=_bc, bread=b) #, learnmore=lm)
+    return render_template('single.html', kid='repn.about', title=t, body_class=_bc, bread=b)  # , learnmore=lm)
 
 # @app.route("/Representation/history")
+
+
 def representations_history():
     t = 'Representations'
     b = [(t, url_for('representations'))]
     b.append(('History', url_for("representations_history")))
     return render_template(_single_knowl, title="A brief history of representations", kid='repn.history', body_class=_bc, bread=b)
 
+
 @app.route('/Motive')
 @app.route('/Motive/')
 def motives():
     t = 'Motives'
     b = [(t, url_for('motives'))]
     # lm = [('History of motives', '/Motives/history')]
-    return render_template('single.html', kid='motives.about', title=t, body_class=_bc, bread=b) #, learnmore=lm)
+    return render_template('single.html', kid='motives.about', title=t, body_class=_bc, bread=b)  # , learnmore=lm)
 
 # @app.route("/Motives/history")
+
+
 def motives_history():
     t = 'Motives'
     b = [(t, url_for('motives'))]
     b.append(('History', url_for("motives_history")))
     return render_template(_single_knowl, title="A brief history of motives", kid='motives.history', body_class=_bc, bread=b)
 
+
 @app.route('/Group')
 @app.route('/Group/')
 def groups():
     t = 'Groups'
     b = [(t, url_for('groups'))]
     # lm = [('History of groups', '/Group/history')]
-    return render_template('single.html', kid='group.about', title=t, body_class=_bc, bread=b) #, learnmore=lm)
+    return render_template('single.html', kid='group.about', title=t, body_class=_bc, bread=b)  # , learnmore=lm)
 
 # @app.route("/Group/history")
+
+
 def groups_history():
     t = 'Groups'
     b = [(t, url_for('groups'))]
     b.append(('History', url_for("groups_history")))
     return render_template(_single_knowl, title="A brief history of groups", kid='group.history', body_class=_bc, bread=b)
 
+
 @app.route("/editorial-board")
 @app.route("/management-board")
 @app.route("/management")
@@ -481,6 +545,7 @@ def editorial_board():
     b = [(t, url_for("editorial_board"))]
     return render_template('management.html', title=t, bread=b)
 
+
 @app.route("/citation")
 def citation():
     t = "Citing the LMFDB"
@@ -494,6 +559,7 @@ def contact():
     b = [(t, url_for("contact"))]
     return render_template('contact.html', title=t, body_class='', bread=b)
 
+
 def root_static_file(name):
     def static_fn():
         fn = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static", name)
@@ -522,10 +588,13 @@ def robots_txt():
     return "User-agent: *\nDisallow: / \n"
 
 # geeky pages have humans.txt
+
+
 @app.route("/humans.txt")
 def humans_txt():
     return render_template("acknowledgment.html", title="Acknowledgments")
 
+
 @app.context_processor
 def add_colors():
     # FIXME:
@@ -554,6 +623,7 @@ def add_colors():
             color = Configuration().get_color()
     return dict(color=all_color_schemes[color].dict())
 
+
 @app.route("/style.css")
 def css():
     response = make_response(render_template("style.css"))
@@ -565,16 +635,19 @@ def css():
         response.headers['Cache-Control'] = 'public, max-age=600'
     return response
 
+
 @app.route("/not_yet_implemented")
 def not_yet_implemented():
     return render_template("not_yet_implemented.html", title="Not Yet Implemented")
 
 # the checklist is used for human testing on a high-level, supplements test.sh
 
+
 @app.route("/checklist-list")
 def checklist_list():
     return render_template("checklist.html", body_class="checklist")
 
+
 @app.route("/checklist")
 def checklist():
     return render_template("checklist-fs.html")
@@ -583,9 +656,12 @@ def checklist():
 #         Intro pages        #
 ##############################
 
+
 # common base class and bread
 _bc = 'intro'
-intro_bread = lambda: [('Intro', url_for("introduction"))]
+def intro_bread():
+    return [('Intro', url_for("introduction"))]
+
 
 # template displaying just one single knowl as an KNOWL_INC
 _single_knowl = 'single.html'
@@ -596,6 +672,7 @@ def introduction():
     b = intro_bread()
     return render_template(_single_knowl, title="Introduction", kid='intro', body_class=_bc, bread=b)
 
+
 @app.route("/intro/features")
 def introduction_features():
     b = intro_bread()
@@ -609,16 +686,19 @@ def introduction_zetatour():
     b.append(('Tutorial', url_for("introduction_zetatour")))
     return render_template(_single_knowl, title="A tour of the Riemann zeta function", kid='intro.tutorial', body_class=_bc, bread=b)
 
+
 @app.route("/bigpicture")
 def bigpicture():
     b = [('Big picture', url_for('bigpicture'))]
     return render_template("bigpicture.html", title="A map of the LMFDB", body_class=_bc, bread=b)
 
+
 @app.route("/universe")
 def universe():
     b = [('LMFDB universe', url_for('universe'))]
     return render_template("universe.html", title="The LMFDB universe", body_class=_bc, bread=b)
 
+
 @app.route("/news")
 def news():
     t = "News"
@@ -626,13 +706,11 @@ def news():
     return render_template(_single_knowl, title="LMFDB in the news", kid='doc.news.in_the_news', body_class=_bc, bread=b)
 
 
-
-
 ###############################################
 # White listing routes for www.lmfdb.org      #
 ###############################################
 
-
+@cached_function
 def routes():
     """
     Returns all routes
@@ -647,7 +725,8 @@ def routes():
             except Exception:
                 url = None
             links.append((url, str(rule)))
-    return sorted(links, key= lambda elt: elt[1])
+    return sorted(links, key=lambda elt: elt[1])
+
 
 @app.route("/sitemap")
 def sitemap():
@@ -667,102 +746,109 @@ def sitemap():
         + "</ul>"
     )
 
-WhiteListedRoutes = [
-    'ArtinRepresentation',
-    'Character/Dirichlet',
-    'Character/calc-gauss/Dirichlet',
-    'Character/calc-jacobi/Dirichlet',
-    'Character/calc-kloosterman/Dirichlet',
-    'Character/calc-value/Dirichlet',
-    'EllipticCurve',
-    'Field',
-    'GaloisGroup',
-    'Genus2Curve/Q',
-    'Group',
-    'HigherGenus/C/Aut',
-    'L/Completeness',
-    'L/CuspForms',
-    'L/Labels',
-    'L/Lhash',
-    'L/Plot',
-    'L/Riemann',
-    'L/SymmetricPower',
-    'L/contents',
-    'L/degree',
-    'L/download',
-    'L/history',
-    'L/interesting',
-    'L/lhash',
-    'L/rational',
-    'L/tracehash',
-    'L/download',
-    'LocalNumberField',
-    'LuCaNT',
-    'ModularForm/GL2/ImaginaryQuadratic',
-    'ModularForm/GL2/Q/Maass',
-    'ModularForm/GL2/Q/holomorphic',
-    'ModularForm/GL2/TotallyReal',
-    'NumberField',
-    'Representation/foo',  # allows /Representation but not /Representation/Galois/ModL/
-    'SatoTateGroup',
-    'Variety/Abelian/Fq',
-    'about',
-    'acknowledgment',
-    'alive',
-    'api',
-    'api2',
-    'bigpicture',
-    'callback_ajax',
-    'citation',
-    'contact',
-    'editorial-board',
-    'favicon.ico',
-    'features',
-    'forcebetasitemap',
-    'health',
-    'humans.txt',
-    'info',
-    'intro',
-    'inventory',
-    'knowledge',
-    'lucant',
-    'management',
-    'padicField',
-    'news',
-    'not_yet_implemented',
-    'random',
-    'robots.txt',
-    'search',
-    'sitemap',
-    'static',
-    'statshealth',
-    'style.css',
-    'universe',
-    'users',
-    'whitelistedsitemap',
-    'zeros/zeta'
-]
-
-WhiteListedBreads = set()
-for elt in WhiteListedRoutes:
-    elt_split = elt.split('/')
-    bread = ''
-    for s in elt.split('/'):
-        if bread:
-            bread += '/' + s
-        else:
-            bread = s
-        WhiteListedBreads.add(bread)
-
 
+@cached_function
+def WhiteListedRoutes():
+    return [
+        'ArtinRepresentation',
+        'Character/Dirichlet',
+        'Character/calc-gauss/Dirichlet',
+        'Character/calc-jacobi/Dirichlet',
+        'Character/calc-kloosterman/Dirichlet',
+        'Character/calc-value/Dirichlet',
+        'EllipticCurve',
+        'Field',
+        'GaloisGroup',
+        'Genus2Curve/Q',
+        'Group',
+        'HigherGenus/C/Aut',
+        'L/Completeness',
+        'L/CuspForms',
+        'L/Labels',
+        'L/Lhash',
+        'L/Plot',
+        'L/Riemann',
+        'L/SymmetricPower',
+        'L/contents',
+        'L/degree',
+        'L/download',
+        'L/history',
+        'L/interesting',
+        'L/lhash',
+        'L/rational',
+        'L/tracehash',
+        'L/download',
+        'LocalNumberField',
+        'LuCaNT',
+        'ModularForm/GL2/ImaginaryQuadratic',
+        'ModularForm/GL2/Q/Maass',
+        'ModularForm/GL2/Q/holomorphic',
+        'ModularForm/GL2/TotallyReal',
+        'NumberField',
+        'Representation/foo',  # allows /Representation but not /Representation/Galois/ModL/
+        'SatoTateGroup',
+        'Variety/Abelian/Fq',
+        'about',
+        'acknowledgment',
+        'alive',
+        'api',
+        'api2',
+        'bigpicture',
+        'callback_ajax',
+        'citation',
+        'contact',
+        'editorial-board',
+        'favicon.ico',
+        'features',
+        'forcebetasitemap',
+        'health',
+        'humans.txt',
+        'info',
+        'intro',
+        'inventory',
+        'knowledge',
+        'lucant',
+        'management',
+        'padicField',
+        'news',
+        'not_yet_implemented',
+        'random',
+        'robots.txt',
+        'search',
+        'sitemap',
+        'static',
+        'statshealth',
+        'style.css',
+        'universe',
+        'users',
+        'whitelistedsitemap',
+        'zeros/zeta'
+    ]
+
+
+@cached_function
+def WhiteListedBreads():
+    res = set()
+    for elt in WhiteListedRoutes():
+        bread = ''
+        for s in elt.split('/'):
+            if bread:
+                bread += '/' + s
+            else:
+                bread = s
+            res.add(bread)
+    return res
+
+
+@cached_function
 def white_listed(url):
     url = url.rstrip("/").lstrip("/")
     if not url:
         return True
     if (
-        any(url.startswith(elt) for elt in WhiteListedRoutes)
+        any(url.startswith(elt) for elt in WhiteListedRoutes())
         # check if is an allowed bread
-        or url in WhiteListedBreads
+        or url in WhiteListedBreads()
     ):
         return True
     # check if it starts with an L
@@ -774,6 +860,24 @@ def white_listed(url):
         return False
 
 
+@cached_function
+def NotWhiteListedBreads():
+    res = set()
+    for _, endpoint in routes():
+        print(endpoint)
+        if not white_listed(endpoint):
+            print(endpoint.lstrip("/").split('/', 1))
+            res.add(endpoint.lstrip("/").split('/', 1)[0])
+    print(res)
+    return res
+
+
+@cached_function
+def valid_bread(url):
+    url = url.lstrip("/")
+    return url.split('/', 1)[0] in NotWhiteListedBreads()
+
+
 @app.route("/forcebetasitemap")
 def forcebetasitemap():
     """
@@ -783,14 +887,14 @@ def forcebetasitemap():
         "<ul>"
         + "\n".join(
             [
-                '<li><a href="{0}">{1}</a></li>'.format(url, endpoint)
+                '<li><a href="{0}">{1}</a></li>'.format(escape(url), escape(endpoint))
                 if url is not None
-                else "<li>{0}</li>".format(endpoint)
+                else "<li>{0}</li>".format(escape(endpoint))
                 for url, endpoint in routes()
-                if not white_listed(endpoint)
+                if not white_listed(endpoint) and valid_bread(endpoint)
             ]
         )
-        + "</ul>"
+        + "</ul>" + str(NotWhiteListedBreads())
     )
 
 
@@ -803,13 +907,12 @@ def whitelistedsitemap():
         "<ul>"
         + "\n".join(
             [
-                '<li><a href="{0}">{1}</a></li>'.format(url, endpoint)
+                '<li><href="{0}">{1}</a></li>'.format(escape(url), escape(endpoint))
                 if url is not None
-                else "<li>{0}</li>".format(endpoint)
+                else "<li>{0}</li>".format(escape(endpoint))
                 for url, endpoint in routes()
                 if white_listed(endpoint)
             ]
         )
         + "</ul>"
     )
-
diff --git a/lmfdb/genus2_curves/__init__.py b/lmfdb/genus2_curves/__init__.py
--- a/lmfdb/genus2_curves/__init__.py
+++ b/lmfdb/genus2_curves/__init__.py
@@ -5,8 +5,7 @@
 from flask import Blueprint
 from lmfdb.api2.searchers import register_search_function
 
-g2c_page = Blueprint("g2c", __name__, template_folder='templates',
-        static_folder="static")
+g2c_page = Blueprint("g2c", __name__, template_folder='templates')
 g2c_logger = make_logger(g2c_page)
 
 @g2c_page.context_processor


Test patch: