Repository: osome-iu/osometweet (language: python)

Issue:
Functions for data cleaning
I think it would be good to build out functions for cleaning the data that `osometweet` returns. 

I was watching one of the Twitter Twitch sessions about doing this and they shared [this module from twarc](https://github.com/DocNow/twarc/blob/v2/twarc/expansions.py) which can be used to partially flatten the data JSON to create a data frame for a csv file. I don't think we can directly take this and work it into our package, however, I thought it could be good to reference. 

I'm thinking that this would require it's own module, maybe called `wrangle` or `data`? Open to suggestions. I'm thinking of functions like:
- `wrangle.extract()` which takes an input of a raw tweet dict and then "text", "urls", "hashtags", etc.
- `wrangle.flatten()` which takes a single raw tweet dict and returns it in row form
- etc.




Discussion / hints:

Fix patch:
diff --git a/osometweet/wrangle.py b/osometweet/wrangle.py
new file mode 100644
index 0000000..025e765
--- /dev/null
+++ b/osometweet/wrangle.py
@@ -0,0 +1,226 @@
+"""
+A collection of convenience functions for manipulating data.
+"""
+
+from collections.abc import MutableMapping
+
+def _flatten_dict_gen(d, parent_key, sep):
+    for k, v in d.items():
+        new_key = parent_key + sep + k if parent_key else k
+        if isinstance(v, MutableMapping):
+            yield from flatten_dict(v, new_key, sep=sep).items()
+        else:
+            yield new_key, v
+
+
+def flatten_dict(dictionary: dict, parent_key: str = '', sep: str = '.'):
+    """
+    Flatten a nested dictionary (such as a Twitter data object).
+
+    For example, convert the dictionary:
+        {"a":{"b":{"c":1234}}}
+
+    into:
+        {'a.b.c': 1234}
+
+    Parameters:
+    ----------
+    - dictionary (dict) : A dictionary object to flatten
+    - parent_key (str) : The base string that will prefix all
+        keys. Typically, left as and empty string (i.e., '')
+        unless you know what you're doing.
+    - sep (str) : The text you would like to separate key path items.
+        Default is a period (i.e., ".")
+
+    Returns:
+    ----------
+    - flat_dict (dict) : a flattened version of `dictionary`
+
+    Raises:
+    ----------
+    - TypeError
+
+    ---------
+    Examples:
+
+    # Create dictionary
+    dictionary = {
+        "a" : 1,
+        "b" : {
+            "c" : 2,
+            "d" : 5
+        },
+        "e" : {
+            "f" : 4,
+            "g" : 3
+        },
+        "h" : 3
+    }
+
+    ### 1. Using function as is
+    flatten_dict(dictionary)
+
+    # Returns:
+    {'a': 1, 'b.c': 2, 'b.d': 5, 'e.f': 4, 'e.h': 3, 'i': 3}
+
+    ~~~
+    ### 2. Changing `parent_key`
+    # Parent key will add `parent_key` as a prefix to all keys
+    flatten_dict(dictionary, parent_key = "NEW")
+
+    # Returns
+    {'NEW.a': 1, 'NEW.b.c': 2, 'NEW.b.d': 5, 'NEW.e.f': 4, 'NEW.e.h': 3, 'NEW.i': 3}
+
+    ~~~
+    ### 3. Changing `sep`
+    # This string is what will separate key path strings
+    flatten_dict(dictionary, sep = "/")
+
+    # Returns
+    {'a': 1, 'b/c': 2, 'b/d': 5, 'e/f': 4, 'e/h': 3, 'i': 3}
+
+    """
+    if not isinstance(dictionary, dict):
+        raise TypeError("`dictionary` must be a dictionary (i.e., type(dictionary) == dict)")
+    flat_dict = dict(_flatten_dict_gen(dictionary, parent_key, sep))
+    return flat_dict
+
+
+def get_dict_paths(dictionary: dict, path: list = []):
+    """
+    Return a generator which iterates over all full
+    key paths within `dictionary`.
+
+    Parameters:
+    ----------
+    - dictionary (dict) : this is the dictionary you'd like to pass
+    - path (list) : Typically left as an empty list. If not, the dictionary
+        key paths will all append to the provided list.
+
+    Returns:
+    ----------
+    - generator
+
+    Raises:
+    ----------
+    - TypeError
+
+    Example:
+    ----------
+
+    # Create dictionary
+    dictionary = {
+        "a" : 1,
+        "b" : {
+            "c" : 2,
+            "d" : 5
+        },
+        "e" : {
+            "f" : 4,
+            "g" : 3
+        },
+        "h" : 3
+    }
+
+    # Call get_dict_paths
+    print(list(get_dict_paths(dictionary)))
+
+    # Returns
+    [['a'], ['b', 'c'], ['b', 'd'], ['e', 'f'], ['e', 'g'], ['h']]
+
+    """
+    if not isinstance(path, list):
+        raise TypeError("`path` must be a list")
+
+    if not isinstance(dictionary, dict):
+        yield path
+    else:
+        yield from [
+            new for key, val in dictionary.items()
+            for new in get_dict_paths(val, path+[key])
+        ]
+
+def get_dict_val(dictionary: dict, key_list: list = []):
+    """
+    Return `dictionary` value at the end of the key path provided
+    in `key_list`.
+
+    Indicate what value to return based on the key_list provided.
+    For example, from left to right, each string in the key_list
+    indicates another nested level further down in the dictionary.
+    If no value is present, a `None` is returned.
+
+    Parameters:
+    ----------
+    - dictionary (dict) : the dictionary object to traverse
+    - key_list (list) : list of strings indicating what dict_obj
+        item to retrieve
+
+    Returns:
+    ----------
+    - key value (if present) or None (if not present)
+
+    Raises:
+    ----------
+    - TypeError
+
+    Examples:
+    ---------
+    # Create dictionary
+    dictionary = {
+        "a" : 1,
+        "b" : {
+            "c" : 2,
+            "d" : 5
+        },
+        "e" : {
+            "f" : 4,
+            "g" : 3
+        },
+        "h" : 3
+    }
+
+    ### 1. Finding an existing value
+    # Create key_list
+    key_list = ['b', 'c']
+
+    # Execute function
+    get_dict_val(dictionary, key_list)
+
+    # Returns
+    2
+
+    ~~~
+
+    ### 2. When input key_path doesn't exist
+
+    # Create key_list
+    key_list = ['b', 'k']
+
+    # Execute function
+    value = get_dict_val(dictionary, key_list)
+
+    # Returns NoneType because the provided path doesn't exist
+    type(value)
+    NoneType
+    """
+
+    if not isinstance(dictionary, dict):
+        raise TypeError("`dictionary` must be of type `dict`")
+
+    if not isinstance(key_list, list):
+        raise TypeError("`key_list` must be of type `list`")
+
+    retval = dictionary
+    for k in key_list:
+
+        # If retval is not a dictionary, we're going too deep
+        if not isinstance(retval, dict):
+            return None
+
+        if k in retval:
+            retval = retval[k]
+
+        else:
+            return None
+    return retval


Test patch:
diff --git a/tests/tests.py b/tests/tests.py
index d32d8aa..5e0f649 100644
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -2,6 +2,7 @@
 import os
 import unittest
 import osometweet
+import osometweet.wrangle
 
 api_key = os.environ.get('TWITTER_API_KEY', '')
 api_key_secret = os.environ.get('TWITTER_API_KEY_SECRET', '')
@@ -27,7 +28,7 @@ def test_1a(self):
             ).json()
         self.assertEqual(resp['data'][0]['id'], test_tweet_id)
         oauth1a._oauth_1a.close()
-    
+
     def test_1a_exception(self):
         with self.assertRaises(ValueError):
             osometweet.OAuth1a(api_key=1)
@@ -37,7 +38,7 @@ def test_1a_exception(self):
             osometweet.OAuth1a(access_token=1)
         with self.assertRaises(ValueError):
             osometweet.OAuth1a(access_token_secret=1)
-    
+
     def test_2(self):
         oauth2 = osometweet.OAuth2(bearer_token=bearer_token)
         test_tweet_id = '1323314485705297926'
@@ -46,7 +47,7 @@ def test_2(self):
             {"ids": f"{test_tweet_id}"}
             ).json()
         self.assertEqual(resp['data'][0]['id'], test_tweet_id)
-    
+
     def test_2_exception(self):
         with self.assertRaises(ValueError):
             osometweet.OAuth2(bearer_token=1)
@@ -65,19 +66,19 @@ def test_tweet_lookup(self):
         resp = self.ot.tweet_lookup(tids=test_tweet_ids)
         for tweet in resp['data']:
             self.assertIn(tweet['id'], test_tweet_ids)
-    
+
     def test_user_lookup_ids(self):
         test_user_ids = ['12', '13']
         resp = self.ot.user_lookup_ids(test_user_ids)
         for user in resp['data']:
-            self.assertIn(user['id'], test_user_ids) 
+            self.assertIn(user['id'], test_user_ids)
 
     def test_user_lookup_usernames(self):
         test_user_usernames = ['jack', 'biz']
         resp = self.ot.user_lookup_usernames(test_user_usernames)
         for user in resp['data']:
-            self.assertIn(user['username'], test_user_usernames) 
-    
+            self.assertIn(user['username'], test_user_usernames)
+
     def test_get_followers(self):
         resp = self.ot.get_followers('12')
         self.assertEqual(resp['meta']['result_count'], len(resp['data']))
@@ -188,7 +189,7 @@ class TestExpansions(unittest.TestCase):
     def setUp(self):
         oauth2 = osometweet.OAuth2(bearer_token=bearer_token)
         self.ot = osometweet.OsomeTweet(oauth2)
-    
+
     def test_tweet_expansions(self):
         expansions_to_request = [
             "attachments.media_keys", "referenced_tweets.id", "author_id"
@@ -230,5 +231,40 @@ def test_chunker(self):
             self.assertEqual(resp, correct_resp)
 
 
+class TestWranlge(unittest.TestCase):
+    """
+    Test all wrangle package methods
+    """
+    def setUp(self):
+        self.wrangle = osometweet.wrangle
+        self._dictionary = {'a': 1, 'b': {'c': 2, 'd': 5}, 'e': {'f': 4, 'g': 3}, 'h': 3, 'i': 'jk'}
+        self._flat_dict1 = {'a': 1, 'b.c': 2, 'b.d': 5, 'e.f': 4, 'e.g': 3, 'h': 3, 'i': 'jk'}
+        self._flat_dict2 = {'a': 1, 'b/c': 2, 'b/d': 5, 'e/f': 4, 'e/g': 3, 'h': 3, 'i': 'jk'}
+        self._key_paths = [['a'], ['b', 'c'], ['b', 'd'], ['e', 'f'], ['e', 'g'], ['h'], ['i']]
+
+    def test_flatten_dict(self):
+        flat_dict1 = self.wrangle.flatten_dict(self._dictionary)
+        self.assertEqual(flat_dict1, self._flat_dict1)
+
+        flat_dict2 = self.wrangle.flatten_dict(
+            self._dictionary,
+            sep = "/"
+            )
+        self.assertEqual(flat_dict2, self._flat_dict2)
+
+    def test_get_dict_paths(self):
+        key_paths = self.wrangle.get_dict_paths(self._dictionary)
+        self.assertEqual(list(key_paths), self._key_paths)
+
+    def test_get_dict_val(self):
+        value = self.wrangle.get_dict_val(self._dictionary, self._key_paths[1])
+        self.assertEqual(value, 2)
+
+        value = self.wrangle.get_dict_val(self._dictionary, ['i'])
+        self.assertEqual(value, 'jk')
+
+        value = self.wrangle.get_dict_val(self._dictionary, ['i', 'j'])
+        self.assertEqual(value, None)
+
 if __name__ == "__main__":
     unittest.main()
\ No newline at end of file