Repository: google/personfinder

Issue:
Sitemap and sitemap ping not working
I found that sitemap and sitemap ping of Person Finder are not working.

And it looks like it is failing for long, at least for 7 years.

Maybe we can just remove it because Google probably does a good job finding pages quickly without sitemap, and we actually have seen no issues in these 7 years? WDYT? @nworden 

One potential counterargument is that it may still help Google to find a new repository even more quickly.

I noticed that because "sitemap ping" cron job is failing in our production. Also https://google.org/personfinder/japan/sitemap returns an error.

I found multiple problems:
* The URL specified in the cron job, /sitemap/ping?search_engine=google, is 404. It misses a repository name.
* /sitemap fails due to out of memory.
* The datastore contained two entities of StaticSiteMapInfo, both created at 2011, while the code expects only one entity. I have just deleted one of them.



Discussion:

Resolving patch:
diff --git a/app/admin.py b/app/admin.py
--- a/app/admin.py
+++ b/app/admin.py
@@ -20,6 +20,7 @@
 
 from const import *
 from model import *
+import sitemap
 from utils import *
 import const
 import tasks
@@ -119,6 +120,7 @@ def post(self):
                         'launch_status',
                         'test_mode',
                     ]):
+                sitemap.SiteMapPing.add_ping_tasks()
                 self.redirect('/admin')
 
         elif self.params.operation == 'save_global':
diff --git a/app/sitemap.py b/app/sitemap.py
--- a/app/sitemap.py
+++ b/app/sitemap.py
@@ -15,142 +15,77 @@
 
 """Exports the URLs of all person entries to a sitemap.xml file."""
 
-__author__ = 'jocatalano@google.com (Joe Catalano) and many other Googlers'
-
 import logging
+import requests
+import requests_toolbelt.adapters.appengine
+import time
+import urllib
 
 from datetime import datetime, timedelta
-from google.appengine.api import urlfetch
-from model import *
-from time import *
-from utils import *
-
-def _compute_max_shard_index(now, sitemap_epoch, shard_size_seconds):
-    delta = now - sitemap_epoch
-    delta_seconds = delta.days * 24 * 60 * 60 + delta.seconds
-    return delta_seconds / shard_size_seconds
-
-def _get_static_sitemap_info(repo):
-    infos = StaticSiteMapInfo.all().fetch(2)
-    if len(infos) > 1:
-        logging.error("There should be at most 1 StaticSiteMapInfo record!")
-        return None
-    elif len(infos) == 1:
-        return infos[0]
-    else:
-        # Set the sitemap generation time according to the time of the first
-        # record with a timestamp.    This will make the other stuff work
-        # correctly in case there is no static sitemap.
-        query = Person.all_in_repo(repo)
-        query = query.filter('last_modified != ', None)
-        first_updated_person = query.order('last_modified').get()
-        if not first_updated_person:
-            # No records; set the time to now.
-            time = get_utcnow()
-        else:
-            # Set the time to just before the first person was entered.
-            time = first_updated_person.last_modified - timedelta(seconds=1)
-        info = StaticSiteMapInfo(static_sitemaps_generation_time=time)
-        db.put(info)
-        return info
+from google.appengine.api import taskqueue
+
+import const
+from model import Repo
+from utils import BaseHandler
+
+
+# Use the App Engine Requests adapter. This makes sure that Requests uses
+# URLFetch.
+# TODO(nworden): see if we should condition this on the runtime (Python 2 vs. 3)
+requests_toolbelt.adapters.appengine.monkeypatch()
+
 
 class SiteMap(BaseHandler):
-    _FETCH_LIMIT = 1000
+
+    repo_required = False
 
     def get(self):
-        requested_shard_index = self.request.get('shard_index')
-        sitemap_info = _get_static_sitemap_info(self.repo)
-        shard_size_seconds = sitemap_info.shard_size_seconds
-        then = sitemap_info.static_sitemaps_generation_time
-
-        if not requested_shard_index:
-            max_shard_index = _compute_max_shard_index(
-                get_utcnow(), then, shard_size_seconds)
-            shards = []
-            for shard_index in range(max_shard_index + 1):
-                shard = {}
-                shard['index'] = shard_index
-                offset_seconds = shard_size_seconds * (shard_index + 1)
-                shard['lastmod'] = format_sitemaps_datetime(
-                    then + timedelta(seconds=offset_seconds))
-                shards.append(shard)
-            self.render('sitemap-index.xml', shards=shards,
-                        static_lastmod=format_sitemaps_datetime(then),
-                        static_map_files=sitemap_info.static_sitemaps)
-        else:
-            shard_index = int(requested_shard_index)
-            assert 0 <= shard_index < 50000    #TODO: nicer error (400 maybe)
-            persons = []
-            time_lower = \
-                then + timedelta(seconds=shard_size_seconds * shard_index)
-            time_upper = time_lower + timedelta(seconds=shard_size_seconds)
-            query = Person.all_in_repo(self.repo
-                         ).filter('last_modified >', time_lower
-                         ).filter('last_modified <=', time_upper
-                         ).order('last_modified')
-            fetched_persons = query.fetch(self._FETCH_LIMIT)
-            while fetched_persons:
-                persons.extend(fetched_persons)
-                last_value = fetched_persons[-1].last_modified
-                query = Person.all_in_repo(self.repo
-                             ).filter('last_modified >', last_value
-                             ).filter('last_modified <=', time_upper
-                             ).order('last_modified')
-                fetched_persons = query.fetch(self._FETCH_LIMIT)
-            urlinfos = [
-                {'person_record_id': p.record_id,
-                 'lastmod': format_sitemaps_datetime(p.last_modified)}
-                for p in persons]
-            self.render('sitemap.xml', urlinfos=urlinfos)
+        langs = const.LANGUAGE_ENDONYMS.keys()
+        urlpaths = []
+        # Include the root/global homepage.
+        urlpaths.append({lang: '?lang=%s' % lang for lang in langs})
+        # Include the repo homepages.
+        for repo in Repo.list_launched():
+            urlpaths.append({
+                lang: '%s?lang=%s' % (repo, lang) for lang in langs})
+        self.render('sitemap.xml', urlpaths=urlpaths)
+
 
 class SiteMapPing(BaseHandler):
-    """Pings the index server with sitemap files that are new since last ping"""
-    _INDEXER_MAP = {'google': 'http://www.google.com/webmasters/tools/ping?',
-                    'not-specified': ''}
+    """Pings the index server."""
+    _INDEXER_MAP = {
+        'bing': 'http://www.bing.com/ping?sitemap=%s',
+        'google': 'http://www.google.com/ping?sitemap=%s',
+    }
 
-    def get(self):
-        search_engine = self.request.get('search_engine')
-        if not search_engine:
-            search_engine = 'not-specified'
-
-        last_update_query = SiteMapPingStatus.all()
-        last_update_query.filter('search_engine = ', search_engine)
-        last_update_status = last_update_query.fetch(1)
-        if not last_update_status:
-            last_shard = -1
-            last_update_status = SiteMapPingStatus(search_engine=search_engine)
-        else:
-            last_update_status = last_update_status[0]
-            last_shard = last_update_status.shard_index
+    _GET_PARAM_SEARCH_ENGINE = 'search_engine'
+
+    repo_required = False
 
-        sitemap_info = _get_static_sitemap_info(self.repo)
-        generation_time = sitemap_info.static_sitemaps_generation_time
-        shard_size_seconds = sitemap_info.shard_size_seconds
+    @staticmethod
+    def add_ping_tasks():
+        for search_engine in SiteMapPing._INDEXER_MAP:
+            name = 'sitemapping-%s-%s' % (search_engine, int(time.time()*1000))
+            taskqueue.add(
+                name=name, method='GET', url='/global/sitemap/ping',
+                params={SiteMapPing._GET_PARAM_SEARCH_ENGINE: search_engine})
 
-        max_shard_index = _compute_max_shard_index(
-            get_utcnow(), generation_time, shard_size_seconds)
-        if not self.ping_indexer(
-            last_shard+1, max_shard_index, search_engine, last_update_status):
+    def get(self):
+        search_engine = self.request.get(SiteMapPing._GET_PARAM_SEARCH_ENGINE)
+        if not search_engine:
+            self.error(500)
+        if not self.ping_indexer(search_engine):
             self.error(500)
 
-    def ping_indexer(self, start_index, end_index, search_engine, status):
+    def ping_indexer(self, search_engine):
         """Pings the server with sitemap updates; returns True if all succeed"""
-        try:
-            for shard_index in range(start_index, end_index + 1):
-                ping_url = self._INDEXER_MAP[search_engine]
-                sitemap_url = 'http://%s/sitemap?shard_index=%s' % (
-                    self.env.netloc, shard_index)
-                ping_url = ping_url + urlencode({'sitemap': sitemap_url})
-                response = urlfetch.fetch(url=ping_url, method=urlfetch.GET)
-                if not response.status_code == 200:
-                    #TODO(jocatalano): Retry or email haiticrisis on failure.
-                    logging.error('Received %d pinging %s',
-                                  response.status_code, ping_url)
-                    return False
-                else:
-                    status.shard_index = shard_index
+        sitemap_url = 'https://%s/global/sitemap' % self.env.netloc
+        ping_url = self._INDEXER_MAP[search_engine] % urllib.quote(sitemap_url)
+        response = requests.get(ping_url)
+        if response.status_code == 200:
             return True
-        finally:
-            # Always update database to reflect how many the max shard that was
-            # pinged particularly when a DeadlineExceededError is thrown
-            db.put(status)
+        else:
+            #TODO(nworden): Retry or email konbit-personfinder on failure.
+            logging.error('Received %d pinging %s',
+                          response.status_code, ping_url)
+            return False


Test patch:
diff --git a/tests/server_test_cases/read_only_tests.py b/tests/server_test_cases/read_only_tests.py
--- a/tests/server_test_cases/read_only_tests.py
+++ b/tests/server_test_cases/read_only_tests.py
@@ -336,11 +336,9 @@ def test_gadget(self):
 
     def test_sitemap(self):
         """Check the sitemap generator."""
-        doc = self.go('/haiti/sitemap')
-        assert '</sitemapindex>' in doc.content
-
-        doc = self.go('/haiti/sitemap?shard_index=1')
-        assert '</urlset>' in doc.content
+        doc = self.go('/global/sitemap')
+        assert 'haiti?lang=en' in doc.content
+        assert 'haiti?lang=es' in doc.content
 
     def test_config_repo_titles(self):
         doc = self.go('/haiti')
diff --git a/tests/test_send_mail.py b/tests/test_send_mail.py
--- a/tests/test_send_mail.py
+++ b/tests/test_send_mail.py
@@ -24,7 +24,6 @@
 import mox
 import os
 import send_mail
-import test_handler
 import unittest
 import webob
 
diff --git a/tests/test_sitemap.py b/tests/test_sitemap.py
new file mode 100644
--- /dev/null
+++ b/tests/test_sitemap.py
@@ -0,0 +1,39 @@
+from mock import patch
+import unittest
+
+from google.appengine.ext import testbed
+
+import sitemap
+import test_handler
+
+class SitemapTests(unittest.TestCase):
+
+    def setUp(self):
+        self.testbed = testbed.Testbed()
+        self.testbed.activate()
+        self.testbed.init_taskqueue_stub()
+        self.testbed.init_user_stub()
+        self.taskqueue_stub = self.testbed.get_stub(
+            testbed.TASKQUEUE_SERVICE_NAME)
+
+    def tearDown(self):
+        self.testbed.deactivate()
+
+    def testAddPingTasks(self):
+        sitemap.SiteMapPing.add_ping_tasks()
+        tasks = self.taskqueue_stub.get_filtered_tasks()
+        self.assertEqual(len(tasks), 2)
+        self.assertEqual(
+            tasks[0].url, '/global/sitemap/ping?search_engine=bing')
+        self.assertEqual(
+            tasks[1].url, '/global/sitemap/ping?search_engine=google')
+
+    def testPingIndexer(self):
+        with patch('requests.get') as requests_mock:
+            handler = test_handler.initialize_handler(
+                sitemap.SiteMapPing, 'sitemap/ping')
+            handler.ping_indexer('google')
+            assert len(requests_mock.call_args_list) == 1
+            call_args, _ = requests_mock.call_args_list[0]
+            assert call_args[0] == ('http://www.google.com/ping?sitemap='
+                                    'https%3A//localhost/global/sitemap')
diff --git a/tests/unit_tests.py b/tests/unit_tests.py
--- a/tests/unit_tests.py
+++ b/tests/unit_tests.py
@@ -33,6 +33,8 @@
 
 # An application id is required to access the datastore, so let's create one
 os.environ['APPLICATION_ID'] = 'personfinder-unittest'
+# The requests library wants this to be set for some reason.
+os.environ['SERVER_SOFTWARE'] = 'testing'
 
 # When the appserver is running, the APP_DIR should be the current directory...
 os.chdir(os.environ['APP_DIR'])