Repository: gratipay/gratipay.com

Issue:
NPM sync is broken
https://gratipay.slack.com/archives/C36LJJF9V/p1494580201702422


Discussion:
Looks like we're getting a 'deleted' changed for a package that doesn't exist in the DB (i.e. a 'create' doc hasn't been received?)

![screen shot 2017-05-12 at 3 10 15 pm](https://cloud.githubusercontent.com/assets/3893573/25992740/2805f34e-3725-11e7-899d-93ce6d48cb26.png)


Resolving patch:
diff --git a/gratipay/sync_npm.py b/gratipay/sync_npm.py
--- a/gratipay/sync_npm.py
+++ b/gratipay/sync_npm.py
@@ -52,7 +52,11 @@ def consume_change_stream(stream, db):
             # Decide what to do.
             if change.get('deleted'):
                 package = Package.from_names(NPM, change['id'])
-                assert package is not None  # right?
+                if not package:
+                    # As a result of CouchDB's compaction algorithm, we might
+                    # receive 'deleted' events for docs even if we haven't seen
+                    # the corresponding events for when the doc was created
+                    continue
                 op, kw = package.delete, {}
             else:
                 op = Package.upsert


Test patch:
diff --git a/tests/py/test_sync_npm.py b/tests/py/test_sync_npm.py
--- a/tests/py/test_sync_npm.py
+++ b/tests/py/test_sync_npm.py
@@ -35,14 +35,9 @@ def test_dedupes_emails(self):
 class ConsumeChangeStreamTests(Harness):
 
     def change_stream(self, changes):
-        def change_stream(seq):
-            for i, change in enumerate(changes):
-                if i < seq: continue
-                change['seq'] = i
-                yield change
-        last_seq = sync_npm.get_last_seq(self.db)
-        return change_stream(last_seq)
-
+        for i, change in enumerate(changes):
+            change['seq'] = i
+            yield change
 
     def test_packages_starts_empty(self):
         assert self.db.all('select * from packages') == []
@@ -70,6 +65,9 @@ def test_not_afraid_to_delete_docs(self):
         sync_npm.consume_change_stream(self.change_stream(docs), self.db)
         assert self.db.one('select * from packages') is None
 
+    def test_delete_tolerates_inexistent_packages(self):
+        docs = [{'deleted': True, 'id': 'foo'}]
+        sync_npm.consume_change_stream(self.change_stream(docs), self.db)
 
     def test_even_deletes_package_with_linked_team(self):
 
@@ -85,21 +83,6 @@ def test_even_deletes_package_with_linked_team(self):
         sync_npm.consume_change_stream(self.change_stream(docs), self.db)
         assert self.db.one('select * from packages') is None
 
-
-    def test_picks_up_with_last_seq(self):
-        docs = [ {'doc': {'name': 'foo', 'description': 'Foo.'}}
-               , {'doc': {'name': 'foo', 'description': 'See alice?',
-                  'maintainers': [{'email': 'alice'}]}}
-               , {'doc': {'name': 'foo', 'description': "No, I don't see alice!"}}
-                ]
-        self.db.run('update worker_coordination set npm_last_seq=2')
-        sync_npm.consume_change_stream(self.change_stream(docs), self.db)
-
-        package = self.db.one('select * from packages')
-        assert package.description == "No, I don't see alice!"
-        assert package.emails == []
-
-
     def test_sets_last_seq(self):
         docs = [{'doc': {'name': 'foo', 'description': 'Foo.'}}] * 13
         assert self.db.one('select npm_last_seq from worker_coordination') == -1