Repository: chaoss/augur

Issue:
Secondary tasks never run when repo count is small
**Description:**
When running on the default repos, I noticed the secondary tasks weren't getting run. This is happening because the core collection is finishing really quickly, updating the weight to a large number, and then the next time around there are no new repos to collect so the scheduler is picking all the old core repos to recollect since there is nothing else to collect. In turn the core repos are always collecting whenever the `start_secondary_collection` function gets run.

**How to reproduce:**
1. Create a new database with the default repos
2. Run collection while watching the collection status table
3. See how the core status while go from collecting to success to collecting to success all while the secondary status stays pending

**Expected behavior:**
I think we should only consider repos for recollection when they are a day or more old. This would leave the core status as success for 1 whole day after finishing, so the secondary collection has a chance to start. This is also useful because it means a small instance collecting 30 repos won't recollect each of those repos multiple times a day since there is nothing else to collect



Discussion:
I think the solution to this issue is to have the secondary task hook check the core_data_last_collected column instead of the status. Although, I think it also makes sense to have a hard limit so we don't collect data for repositories too often. 

Resolving patch:
diff --git a/augur/tasks/start_tasks.py b/augur/tasks/start_tasks.py
--- a/augur/tasks/start_tasks.py
+++ b/augur/tasks/start_tasks.py
@@ -6,6 +6,7 @@
 from enum import Enum
 import math
 import numpy as np
+import datetime
 #from celery.result import AsyncResult
 from celery import signature
 from celery import group, chain, chord, signature
@@ -159,7 +160,7 @@ def non_repo_domain_tasks():
         The below functions define augur's collection hooks.
         Each collection hook schedules tasks for a number of repos
     """
-def start_primary_collection(session,max_repo):
+def start_primary_collection(session,max_repo, days_until_collect_again = 1):
 
     #Get list of enabled phases 
     enabled_phase_names = get_enabled_phase_names_from_config(session.logger, session)
@@ -201,17 +202,21 @@ def core_task_success_util_gen(repo_git):
     #Now start old repos if there is space to do so.
     limit -= collection_size
 
-    collected_before = CollectionStatus.core_data_last_collected != None
-
     if limit > 0:
+        #only start repos older than the specified amount of days
+        cutoff_date = datetime.datetime.now() - datetime.timedelta(days=days_until_collect_again)
+
+        collected_before = CollectionStatus.core_data_last_collected != None
+        old_enough = CollectionStatus.core_data_last_collected <= cutoff_date
+
         start_block_of_repos(
             session.logger, session,
-            and_(not_erroed, not_collecting,collected_before),
+            and_(not_erroed, not_collecting,collected_before,old_enough),
             limit, primary_enabled_phases, repos_type="old", sort=core_order
         )
 
 
-def start_secondary_collection(session,max_repo):
+def start_secondary_collection(session,max_repo, days_until_collect_again = 1):
 
     #Get list of enabled phases 
     enabled_phase_names = get_enabled_phase_names_from_config(session.logger, session)
@@ -251,12 +256,16 @@ def secondary_task_success_util_gen(repo_git):
     )
 
     limit -= collection_size
-    collected_before = CollectionStatus.secondary_data_last_collected != None
 
     if limit > 0:
+        cutoff_date = datetime.datetime.now() - datetime.timedelta(days=days_until_collect_again)
+
+        collected_before = CollectionStatus.secondary_data_last_collected != None
+        old_enough = CollectionStatus.core_data_last_collected <= cutoff_date
+
         start_block_of_repos(
             session.logger, session, 
-            and_(primary_collected,not_erroed, not_collecting,collected_before), 
+            and_(primary_collected,not_erroed, not_collecting,collected_before,old_enough), 
             limit, secondary_enabled_phases,
             repos_type="old",
             hook="secondary",
@@ -301,7 +310,7 @@ def facade_clone_update_success_util_gen(repo_git):
 
     facade_augur_collection.start_data_collection()
 
-def start_facade_collection(session,max_repo):
+def start_facade_collection(session,max_repo,days_until_collect_again = 1):
 
     #Deal with secondary collection
     facade_enabled_phases = []
@@ -342,12 +351,16 @@ def facade_task_update_weight_util_gen(repo_git):
     )
 
     limit -= collection_size
-    collected_before = CollectionStatus.facade_data_last_collected != None
 
     if limit > 0:
+        cutoff_date = datetime.datetime.now() - datetime.timedelta(days=days_until_collect_again)
+
+        collected_before = CollectionStatus.facade_data_last_collected != None
+        old_enough = CollectionStatus.core_data_last_collected <= cutoff_date
+
         start_block_of_repos(
             session.logger, session,
-            and_(not_pending,not_failed_clone,not_erroed, not_collecting, not_initializing,collected_before),
+            and_(not_pending,not_failed_clone,not_erroed, not_collecting, not_initializing,collected_before,old_enough),
             limit, facade_enabled_phases,
             repos_type="old",
             hook="facade",


Test patch: