Repository: zooniverse/aggregation-for-caesar (language: python)

Issue:
Add a consensus reducer
The often-used consensus reducer for Caesar would be a nice addition for folks to be able to use for offline analysis.  I acknowledge it seems a shame to duplicate code, but I don't think there's any way to leverage the existing Ruby code to help here directly.


Discussion / hints:
In words what does the consensus reducer do?  What values does it return?
Hi @CKrawczyk --

I'm thinking the consensus reducer should be a clone of what's currently available for Caesar: return the survey / question option with the most votes.

Outputs:
- `most_likely` = option with greatest number of classifications/votes
- `num_votes` = vote count for mostly likely option
- `agreement` = fraction of total votes held by most likely option.
Perhaps the reducer code could even be a Python port of the Caesar model:
https://github.com/zooniverse/caesar/blob/master/app/models/reducers/consensus_reducer.rb

Fix patch:
diff --git a/docs/source/reducers.rst b/docs/source/reducers.rst
index 0208f13f..6ce7087d 100644
--- a/docs/source/reducers.rst
+++ b/docs/source/reducers.rst
@@ -6,6 +6,11 @@ Reducers
 
 ----
 
+.. automodule:: panoptes_aggregation.reducers.question_consensus_reducer
+  :members:
+
+----
+
 .. automodule:: panoptes_aggregation.reducers.slider_reducer
   :members:
 
diff --git a/panoptes_aggregation/reducers/__init__.py b/panoptes_aggregation/reducers/__init__.py
index 01b33db0..d9a8ef42 100644
--- a/panoptes_aggregation/reducers/__init__.py
+++ b/panoptes_aggregation/reducers/__init__.py
@@ -3,6 +3,7 @@
 from .point_reducer_hdbscan import point_reducer_hdbscan
 from .rectangle_reducer import rectangle_reducer
 from .question_reducer import question_reducer
+from .question_consensus_reducer import question_consensus_reducer
 from .survey_reducer import survey_reducer
 from .poly_line_text_reducer import poly_line_text_reducer
 from .dropdown_reducer import dropdown_reducer
@@ -26,6 +27,7 @@
     'point_reducer_hdbscan': point_reducer_hdbscan,
     'rectangle_reducer': rectangle_reducer,
     'question_reducer': question_reducer,
+    'question_consensus_reducer': question_consensus_reducer,
     'shortcut_reducer': shortcut_reducer,
     'survey_reducer': survey_reducer,
     'poly_line_text_reducer': poly_line_text_reducer,
@@ -38,5 +40,5 @@
     'tess_reducer_column': tess_reducer_column,
     'tess_gold_standard_reducer': tess_gold_standard_reducer,
     'text_reducer': text_reducer,
-    'first_n_true_reducer' : first_n_true_reducer
+    'first_n_true_reducer': first_n_true_reducer
 }
diff --git a/panoptes_aggregation/reducers/question_consensus_reducer.py b/panoptes_aggregation/reducers/question_consensus_reducer.py
new file mode 100644
index 00000000..ece16d52
--- /dev/null
+++ b/panoptes_aggregation/reducers/question_consensus_reducer.py
@@ -0,0 +1,62 @@
+'''
+Question Consensus Reducer
+--------------------------
+This module porvides functions to reduce the question task extracts from
+:mod:`panoptes_aggregation.extractors.question_extractor`.
+'''
+from collections import Counter
+from .reducer_wrapper import reducer_wrapper
+
+DEFAULTS = {
+    'pairs': {'default': False, 'type': bool}
+}
+
+
+@reducer_wrapper(defaults_data=DEFAULTS)
+def question_consensus_reducer(data_list, pairs=False, **kwargs):
+    '''Reduce a list of extracted questions into a consensus description dict
+
+    Parameters
+    ----------
+    data_list : list
+        A list of extractions created by
+        :meth:`panoptes_aggregation.extractors.question_extractor.question_extractor`
+    pairs : bool, optional
+        Default `False`. How multiple choice questions are treated.
+        When `True` the set of all choices is treated as a single answer
+
+    Returns
+    -------
+    reduction : dict
+        A dictinary with the following keys
+
+        * `most_likely` : `key` with greatest number of classifications/votes
+        * `num_votes` : vote count for mostly likely `key`
+        * `agreement` : fraction of total votes held by most likely `key`.
+
+    '''
+    # reduce data (similar to question_reducer)
+    answer_list = []
+    for data in data_list:
+        if pairs:
+            answer_list.append('+'.join(sorted(data)))
+        else:
+            answer_list += list(data)
+    counter_total = Counter(answer_list)
+
+    # default return value
+    result = {
+        "num_votes": 0,
+    }
+    most_common = counter_total.most_common(1)
+    # if there exists a most common element (i.e. keys are not empty)
+    if len(most_common) > 0 and len(most_common[0]) == 2 and most_common[0][1] > 0:
+        max_key = most_common[0][0]
+        # sum the values of every key iff there's a max key that has a value > 0
+        summed_counts = sum(dict(counter_total).values())
+        result = {
+            "most_likely": max_key,
+            "num_votes": most_common[0][1],
+            "agreement": most_common[0][1] / summed_counts
+        }
+    return result


Test patch:
diff --git a/panoptes_aggregation/tests/reducer_tests/test_question_consensus_reducer.py b/panoptes_aggregation/tests/reducer_tests/test_question_consensus_reducer.py
new file mode 100644
index 00000000..fd1ede72
--- /dev/null
+++ b/panoptes_aggregation/tests/reducer_tests/test_question_consensus_reducer.py
@@ -0,0 +1,64 @@
+from panoptes_aggregation.reducers.question_consensus_reducer import question_consensus_reducer
+from .base_test_class import ReducerTestNoProcessing
+
+extracted_data = [
+    {'a': 1, 'b': 1},
+    {'a': 1},
+    {'d': 1, 'a': 1},
+    {'b': 1, 'c': 1},
+    {'b': 1, 'a': 1}
+]
+
+reduced_data = {
+    "most_likely": 'a',
+    "num_votes": 4,
+    "agreement": 4 / 9
+}
+
+TestQuestionReducer = ReducerTestNoProcessing(
+    question_consensus_reducer,
+    extracted_data,
+    reduced_data,
+    'Test question reducer',
+    test_name='TestQuestionConsensusReducer'
+)
+
+reduced_data_pairs = {
+    "most_likely": 'a+b',
+    "num_votes": 2,
+    "agreement": 2 / 5
+}
+
+TestQuestionReducerPairs = ReducerTestNoProcessing(
+    question_consensus_reducer,
+    extracted_data,
+    reduced_data_pairs,
+    'Test question consensus reducer as pairs',
+    kwargs={'pairs': True},
+    test_name='TestQuestionConsensusReducerPairs'
+)
+
+reduced_data_no_votes = {
+    "num_votes": 0
+}
+
+TestQuestionReducerPairsNoVotes = ReducerTestNoProcessing(
+    question_consensus_reducer,
+    [],
+    reduced_data_no_votes,
+    'Test question consensus reducer with no votes',
+    test_name='TestQuestionConsensusReducerNoVotes'
+)
+
+reduced_data_no_votes_pairs = {
+    "num_votes": 0
+}
+
+TestQuestionReducerPairsNoVotesPairs = ReducerTestNoProcessing(
+    question_consensus_reducer,
+    [],
+    reduced_data_no_votes_pairs,
+    'Test question consensus reducer with no votes as pairs',
+    kwargs={'pairs': True},
+    test_name='TestQuestionConsensusReducerNoVotesPairs'
+)