Repository: chaoss/augur

Issue:
Collection Weight doesn't penalize already collected repos until 1 day later 
**Description:**
- The collection weight isn't penalizing repos that have already been collected until a day after there first collection (when the update collection weight task is run). This will mean that really small repos will be collected multiple times in the same day util eventually the update collection weight task is completed

**How to reproduce:**
1. Run augur with a small number of repos
2. Watch the core_weight and see that it doesn't increase after a repo has been collected

**Expected behavior:**
A repo's weight should be updated right after it is finished collecting


Discussion:

Resolving patch:
diff --git a/augur/tasks/git/facade_tasks.py b/augur/tasks/git/facade_tasks.py
--- a/augur/tasks/git/facade_tasks.py
+++ b/augur/tasks/git/facade_tasks.py
@@ -529,6 +529,7 @@ def facade_phase(repo_git):
         if not limited_run or (limited_run and run_facade_contributors):
             facade_core_collection.append(generate_contributor_sequence(logger,repo_git,session))
 
+        #facade_core_collection.append(git_update_commit_count_weight.si(repo_git))
 
         #These tasks need repos to be cloned by facade before they can work.
         facade_sequence.append(
diff --git a/augur/tasks/start_tasks.py b/augur/tasks/start_tasks.py
--- a/augur/tasks/start_tasks.py
+++ b/augur/tasks/start_tasks.py
@@ -87,7 +87,7 @@ def primary_repo_collect_phase(repo_git):
 
     repo_task_group = group(
         repo_info_task,
-        chain(primary_repo_jobs | core_task_update_weight_util.s(repo_git=repo_git),secondary_repo_jobs,process_contributors.si()),
+        chain(primary_repo_jobs | issue_pr_task_update_weight_util.s(repo_git=repo_git),secondary_repo_jobs,process_contributors.si()),
         #facade_phase(logger,repo_git),
         
         collect_releases.si(repo_git),
@@ -311,6 +311,11 @@ def facade_task_success_util_gen(repo_git):
 
     facade_enabled_phases.append(facade_task_success_util_gen)
 
+    def facade_task_update_weight_util_gen(repo_git):
+        return git_update_commit_count_weight.si(repo_git)
+
+    facade_enabled_phases.append(facade_task_update_weight_util_gen)
+
     active_repo_count = len(session.query(CollectionStatus).filter(CollectionStatus.facade_status == CollectionState.COLLECTING.value).all())
 
     #cutoff_date = datetime.datetime.now() - datetime.timedelta(days=days)
@@ -391,7 +396,7 @@ def augur_collection_update_weights():
             status = repo.collection_status[0]
             raw_count = status.issue_pr_sum
 
-            core_task_update_weight_util([int(raw_count)],repo_git=repo_git,session=session)
+            issue_pr_task_update_weight_util([int(raw_count)],repo_git=repo_git,session=session)
     
         facade_not_pending = CollectionStatus.facade_status != CollectionState.PENDING.value
         facade_not_failed = CollectionStatus.facade_status != CollectionState.FAILED_CLONE.value
diff --git a/augur/tasks/util/collection_util.py b/augur/tasks/util/collection_util.py
--- a/augur/tasks/util/collection_util.py
+++ b/augur/tasks/util/collection_util.py
@@ -111,7 +111,24 @@ def task_failed_util(request,exc,traceback):
         
         session.commit()
     
+
+
+#This task updates the core and secondary weight with the issues and prs already passed in
+@celery.task
+def issue_pr_task_update_weight_util(issue_and_pr_nums,repo_git=None,session=None):
+    from augur.tasks.init.celery_app import engine
+    logger = logging.getLogger(issue_pr_task_update_weight_util.__name__)
+
+    if repo_git is None:
+        return
     
+    if session is not None:
+        update_issue_pr_weights(logger, session, repo_git, sum(issue_and_pr_nums))
+    else:
+        with DatabaseSession(logger,engine=engine) as session:
+            update_issue_pr_weights(logger,session,repo_git,sum(issue_and_pr_nums))
+
+
 @celery.task
 def core_task_success_util(repo_git):
 
@@ -135,12 +152,20 @@ def core_task_success_util(repo_git):
 
         session.commit()
 
-def update_repo_core_weight(logger,session,repo_git,raw_sum):
+        repo_git = repo.repo_git
+        status = repo.collection_status[0]
+        raw_count = status.issue_pr_sum
+
+        #Update the values for core and secondary weight
+        issue_pr_task_update_weight_util([int(raw_count)],repo_git=repo_git,session=session)
+
+#Update the existing core and secondary weights as well as the raw sum of issues and prs
+def update_issue_pr_weights(logger,session,repo_git,raw_sum):
     repo = Repo.get_by_repo_git(session, repo_git)
     status = repo.collection_status[0]
 
     try: 
-        weight = raw_sum#get_repo_weight_core(logger,repo_git)
+        weight = raw_sum
 
         weight -= calculate_date_weight_from_timestamps(repo.repo_added, status.core_data_last_collected)
 
@@ -169,23 +194,6 @@ def update_repo_core_weight(logger,session,repo_git,raw_sum):
 
 
 
-#This task updates the weight with the issues and prs already passed in
-@celery.task
-def core_task_update_weight_util(issue_and_pr_nums,repo_git=None,session=None):
-    from augur.tasks.init.celery_app import engine
-    logger = logging.getLogger(core_task_update_weight_util.__name__)
-
-    if repo_git is None:
-        return
-    
-    if session is not None:
-        update_repo_core_weight(logger, session, repo_git, sum(issue_and_pr_nums))
-    else:
-        with DatabaseSession(logger,engine=engine) as session:
-            update_repo_core_weight(logger,session,repo_git,sum(issue_and_pr_nums))
-
-
-
 @celery.task
 def secondary_task_success_util(repo_git):
 
@@ -209,6 +217,13 @@ def secondary_task_success_util(repo_git):
 
         session.commit()
 
+        #Update the values for core and secondary weight
+        repo_git = repo.repo_git
+        status = repo.collection_status[0]
+        raw_count = status.issue_pr_sum
+
+        issue_pr_task_update_weight_util([int(raw_count)],repo_git=repo_git,session=session)
+
 #Get the weight for each repo for the secondary collection hook.
 def get_repo_weight_secondary(logger,repo_git):
     from augur.tasks.init.celery_app import engine


Test patch: