Repository: dssg/triage

Issue:
Add in feature subsetting
We need two classes, that can be configured via the experiment config (names subject to change):

FeatureGroup

FeatureSubsettingStrategy

These are configurable per experiment, and thus their definition must be YAML-serializable. The former defines a way to create feature groups. Whether it be by table name, time, aggregation, etc. To start we are just implementing table names. The class will take in its configuration in the constructor, and have a method that takes in a feature dictionary and outputs the subset of that feature dictionary.

Lists of groups are paired with a strategy (leave one out/leave one in/use all combinations).  The strategy, given a list of groups, can output an iterator/list of feature dictionaries.

Discussion:

Patch:
diff --git a/example_experiment_config.yaml b/example_experiment_config.yaml
index 19a25844..2b5cd470 100644
--- a/example_experiment_config.yaml
+++ b/example_experiment_config.yaml
@@ -87,6 +87,21 @@ feature_aggregations:
         # A list of different columns to separately group by
         groups:
             - 'entity_id'
+
+# FEATURE GROUPING
+# define how to group features and generate combinations
+# feature_group_definition allows you to create groups/subset of your features
+# by different criteria.
+# for instance, 'tables' allows you to send a list of collate feature tables
+# 'prefix' allows you to specify a list of feature name prefixes
+feature_group_definition:
+    tables: ['prefix_entity_id']
+
+# strategies for generating combinations of groups
+# available: all, leave-one-out
+feature_group_strategies: ['all']
+
+
 # GRID CONFIGURATION
 # The classifier/hyperparameter combinations that should be trained
 #
diff --git a/triage/feature_group_creator.py b/triage/feature_group_creator.py
new file mode 100644
index 00000000..95c82033
--- /dev/null
+++ b/triage/feature_group_creator.py
@@ -0,0 +1,62 @@
+def table_subsetter(config_item, table, features):
+    "Return features matching a given table"
+    if table == config_item:
+        return features
+    else:
+        return []
+
+
+def prefix_subsetter(config_item, table, features):
+    "Return features matching a given prefix"
+    return [
+        feature
+        for feature in features
+        if feature.split('_')[0] == config_item
+    ]
+
+
+def all_subsetter(config_item, table, features):
+    return features
+
+
+class FeatureGroupCreator(object):
+    subsetters = {
+        'tables': table_subsetter,
+        'prefix': prefix_subsetter,
+        'all': all_subsetter,
+    }
+
+    def __init__(self, definition):
+        """
+        Args:
+            definition (dict) rules for generating feature groups
+                Each key must correspond to a key in self.subsetters
+                Each value (a list) must be understood by the subsetter
+        """
+        self.definition = definition
+
+        for subsetter in self.definition.keys():
+            if subsetter not in self.subsetters:
+                raise KeyError('Unknown subsetter %s received', subsetter)
+
+    def subsets(self, feature_dictionary):
+        """Generate subsets of a feature dict
+
+        Args:
+            feature_dictionary (dict) tables and the features contained in each
+
+        Returns: (list) subsets of the feature dictionary
+        """
+        subsets = []
+        for name, config in sorted(self.definition.items()):
+            for config_item in config:
+                subset = {}
+                for table, features in feature_dictionary.items():
+                    matching_features =\
+                        self.subsetters[name](config_item, table, features)
+                    if len(matching_features) > 0:
+                        subset[table] = matching_features
+                subsets.append(subset)
+        if not any(subset for subset in subsets if any(subset)):
+            raise Exception('No matching feature subsets found!')
+        return subsets
diff --git a/triage/feature_group_mixer.py b/triage/feature_group_mixer.py
new file mode 100644
index 00000000..a04fa900
--- /dev/null
+++ b/triage/feature_group_mixer.py
@@ -0,0 +1,46 @@
+def leave_one_out(feature_groups):
+    """For each group, return a copy of all groups excluding that group
+
+    Args:
+        feature_groups (list) The feature groups to apply the strategy to
+
+    Returns: A list of feature dicts
+    """
+    results = []
+    for index_to_exclude in range(0, len(feature_groups)):
+        group_copy = feature_groups.copy()
+        del group_copy[index_to_exclude]
+        feature_dict = {}
+        for group in group_copy:
+            feature_dict.update(group)
+        results.append(feature_dict)
+    return results
+
+
+def all_features(feature_groups):
+    return feature_groups
+
+
+class FeatureGroupMixer(object):
+    """Generates different combinations of feature groups
+    based on a list of strategies"""
+    strategy_lookup = {
+        'leave-one-out': leave_one_out,
+        'all': all_features,
+    }
+
+    def __init__(self, strategies):
+        self.strategies = strategies
+
+    def generate(self, feature_groups):
+        """Apply all strategies to the list of feature groups
+
+        Args:
+            feature_groups (list) A list of feature dicts,
+                each representing a group
+        Returns: (list) of feature dicts
+        """
+        results = []
+        for strategy in self.strategies:
+            results += self.strategy_lookup[strategy](feature_groups)
+        return results
diff --git a/triage/features.py b/triage/features.py
index 5cd16bde..1b8d7c30 100644
--- a/triage/features.py
+++ b/triage/features.py
@@ -1,2 +1,4 @@
 from triage.feature_generators import FeatureGenerator
 from triage.feature_dictionary_creator import FeatureDictionaryCreator
+from triage.feature_group_creator import FeatureGroupCreator
+from triage.feature_group_mixer import FeatureGroupMixer
diff --git a/triage/pipelines/base.py b/triage/pipelines/base.py
index a1547c15..7903b352 100644
--- a/triage/pipelines/base.py
+++ b/triage/pipelines/base.py
@@ -1,6 +1,10 @@
 from triage.db import ensure_db
 from triage.label_generators import BinaryLabelGenerator
-from triage.features import FeatureGenerator, FeatureDictionaryCreator
+from triage.features import \
+    FeatureGenerator,\
+    FeatureDictionaryCreator,\
+    FeatureGroupCreator,\
+    FeatureGroupMixer
 from triage.model_trainers import ModelTrainer
 from triage.predictors import Predictor
 from triage.scoring import ModelScorer
@@ -59,6 +63,14 @@ class PipelineBase(object):
             db_engine=self.db_engine
         )
 
+        self.feature_group_creator = FeatureGroupCreator(
+            self.config.get('feature_group_definition', {'all': [True]})
+        )
+
+        self.feature_group_mixer = FeatureGroupMixer(
+            self.config.get('feature_group_strategies', ['all'])
+        )
+
         self.architect = Architect(
             beginning_of_time=dt_from_str(split_config['beginning_of_time']),
             label_names=['outcome'],
diff --git a/triage/pipelines/local_parallel.py b/triage/pipelines/local_parallel.py
index fb3c6efe..fa0564d2 100644
--- a/triage/pipelines/local_parallel.py
+++ b/triage/pipelines/local_parallel.py
@@ -45,9 +45,13 @@ class LocalParallelPipeline(PipelineBase):
             feature_dates=all_as_of_times,
         )
 
-        feature_dict = self.feature_dictionary_creator\
+        master_feature_dict = self.feature_dictionary_creator\
             .feature_dictionary(feature_tables)
 
+        feature_dicts = self.feature_group_mixer.generate(
+            self.feature_group_creator.subsets(master_feature_dict)
+        )
+
         # 4. create training and test sets
         logging.info('Creating matrices')
         logging.debug('---------------------')
@@ -56,7 +60,7 @@ class LocalParallelPipeline(PipelineBase):
 
         updated_split_definitions = self.architect.chop_data(
             split_definitions,
-            feature_dict
+            feature_dicts
         )
 
         for split in updated_split_definitions:
diff --git a/triage/pipelines/serial.py b/triage/pipelines/serial.py
index febb5fa6..79455689 100644
--- a/triage/pipelines/serial.py
+++ b/triage/pipelines/serial.py
@@ -39,9 +39,13 @@ class SerialPipeline(PipelineBase):
             feature_dates=all_as_of_times,
         )
 
-        feature_dict = self.feature_dictionary_creator\
+        master_feature_dict = self.feature_dictionary_creator\
             .feature_dictionary(feature_tables)
 
+        feature_dicts = self.feature_group_mixer.generate(
+            self.feature_group_creator.subsets(master_feature_dict)
+        )
+
         # 4. create training and test sets
         logging.info('Creating matrices')
         logging.debug('---------------------')
@@ -50,7 +54,7 @@ class SerialPipeline(PipelineBase):
 
         updated_split_definitions = self.architect.chop_data(
             split_definitions,
-            feature_dict
+            feature_dicts
         )
 
         for split in updated_split_definitions:


Test patch:
diff --git a/tests/test_feature_group_creator.py b/tests/test_feature_group_creator.py
new file mode 100644
index 00000000..1a326829
--- /dev/null
+++ b/tests/test_feature_group_creator.py
@@ -0,0 +1,53 @@
+from triage.features import FeatureGroupCreator
+
+
+def test_table_group():
+    group = FeatureGroupCreator(
+        definition={'tables': ['one', 'three']}
+    )
+
+    assert group.subsets({
+        'one': ['col_a', 'col_b', 'col_c'],
+        'two': ['col_a', 'col_b', 'col_c'],
+        'three': ['col_a', 'col_b', 'col_c'],
+    }) == [
+        {'one': ['col_a', 'col_b', 'col_c']},
+        {'three': ['col_a', 'col_b', 'col_c']},
+    ]
+
+
+def test_prefix_group():
+    group = FeatureGroupCreator(
+        definition={'prefix': ['major', 'severe']}
+    )
+
+    assert group.subsets({
+        'one': ['minor_a', 'minor_b', 'minor_c'],
+        'two': ['severe_a', 'severe_b', 'severe_c'],
+        'three': ['major_a', 'major_b', 'major_c'],
+        'four': ['minor_a', 'minor_b', 'minor_c'],
+    }) == [
+        {'three': ['major_a', 'major_b', 'major_c']},
+        {'two': ['severe_a', 'severe_b', 'severe_c']},
+    ]
+
+
+def test_multiple_criteria():
+    group = FeatureGroupCreator(
+        definition={
+            'prefix': ['major', 'severe'],
+            'tables': ['one', 'two'],
+        }
+    )
+
+    assert group.subsets({
+        'one': ['minor_a', 'minor_b', 'minor_c'],
+        'two': ['severe_a', 'severe_b', 'severe_c'],
+        'three': ['major_a', 'major_b', 'major_c'],
+        'four': ['minor_a', 'minor_b', 'minor_c'],
+    }) == [
+        {'three': ['major_a', 'major_b', 'major_c']},
+        {'two': ['severe_a', 'severe_b', 'severe_c']},
+        {'one': ['minor_a', 'minor_b', 'minor_c']},
+        {'two': ['severe_a', 'severe_b', 'severe_c']},
+    ]
diff --git a/tests/test_feature_group_mixer.py b/tests/test_feature_group_mixer.py
new file mode 100644
index 00000000..2edeb1bb
--- /dev/null
+++ b/tests/test_feature_group_mixer.py
@@ -0,0 +1,18 @@
+import itertools
+
+from triage.feature_group_mixer import FeatureGroupMixer
+
+
+def test_feature_group_mixer_leave_one_out():
+    english_numbers = {'one': ['two', 'three'], 'four': ['five', 'six']}
+    letters = {'a': ['b', 'c'], 'd': ['e', 'f']}
+    german_numbers = {'eins': ['zwei', 'drei'], 'vier': ['funf', 'sechs']}
+    feature_groups = [english_numbers, letters, german_numbers]
+
+    result = FeatureGroupMixer(['leave-one-out']).generate(feature_groups)
+    expected = [
+        dict(itertools.chain(letters.items(), german_numbers.items())),
+        dict(itertools.chain(english_numbers.items(), german_numbers.items())),
+        dict(itertools.chain(english_numbers.items(), letters.items())),
+    ]
+    assert result == expected