Repository: pandas-dev/pandas

Issue:
Tags to dummies helper function
from @jreback's clever data alignment trick: http://stackoverflow.com/questions/16637171/pandas-reshaping-data

not sure where this should go, it does come up quite frequently, e.g. the Movielens data set https://raw.github.com/pydata/pydata-book/master/ch02/movielens/movies.dat:

```
df = read_table('https://raw.github.com/pydata/pydata-book/master/ch02/movielens/movies.dat', header=None, sep='::')

In [10]: genres = df[2].str.split('|')

In [11]: genres
Out[11]: 
0      [Animation, Children's, Comedy]
1     [Adventure, Children's, Fantasy]
2                    [Comedy, Romance]
3                      [Comedy, Drama]
4                             [Comedy]
5            [Action, Crime, Thriller]
6                    [Comedy, Romance]
7              [Adventure, Children's]
8                             [Action]
9        [Action, Adventure, Thriller]
10            [Comedy, Drama, Romance]
11                    [Comedy, Horror]
12             [Animation, Children's]
13                             [Drama]
14        [Action, Adventure, Romance]
...
3868                              [Horror]
3869                              [Horror]
3870                              [Horror]
3871                              [Horror]
3872                              [Horror]
3873                              [Comedy]
3874                       [Comedy, Drama]
3875    [Adventure, Animation, Children's]
3876             [Action, Drama, Thriller]
3877                            [Thriller]
3878                              [Comedy]
3879                               [Drama]
3880                               [Drama]
3881                               [Drama]
3882                     [Drama, Thriller]
Name: 2, Length: 3883, dtype: object

In [12]: dummies = genres.apply(lambda x: Series(1, index=x)).fillna(0)

In [13]: dummies[:4].T
Out[13]: 
             0  1  2  3
Action       0  0  0  0
Adventure    0  1  0  0
Animation    1  0  0  0
Children's   1  1  0  0
Comedy       1  0  1  1
Crime        0  0  0  0
Documentary  0  0  0  0
Drama        0  0  0  1
Fantasy      0  1  0  0
Film-Noir    0  0  0  0
Horror       0  0  0  0
Musical      0  0  0  0
Mystery      0  0  0  0
Romance      0  0  1  0
Sci-Fi       0  0  0  0
Thriller     0  0  0  0
War          0  0  0  0
Western      0  0  0  0

```



Discussion:
N.B. this method is a bit slow, something more optimized would be nice

Perhaps get_dummies could at the same time be made a Series method (as well as being top level)... Or does Series.str.get_dummies (also??) make sense for this delim-ing...

I don't think this should be a string method 

you can always s.extract().get_dummies() which will operate in iterables inside the series elements (iow lists)

assinging you for 0.14!

sounds good :)


Resolving patch:
diff --git a/pandas/core/reshape.py b/pandas/core/reshape.py
--- a/pandas/core/reshape.py
+++ b/pandas/core/reshape.py
@@ -941,6 +941,8 @@ def get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False):
     1  0  1    0
     2  0  0    1
 
+    See also ``Series.str.get_dummies``.
+
     """
     # Series avoids inconsistent NaN handling
     cat = Categorical.from_array(Series(data))
diff --git a/pandas/core/strings.py b/pandas/core/strings.py
--- a/pandas/core/strings.py
+++ b/pandas/core/strings.py
@@ -187,7 +187,6 @@ def str_contains(arr, pat, case=True, flags=0, na=np.nan, regex=True):
     else:
         f = lambda x: pat in x
     return _na_map(f, arr, na)
-        
 
 
 def str_startswith(arr, pat, na=np.nan):
@@ -460,6 +459,46 @@ def f(x):
     return result
 
 
+def str_get_dummies(arr, sep='|'):
+    """
+    Split each string by sep and return a frame of dummy/indicator variables.
+
+    Examples
+    --------
+    >>> Series(['a|b', 'a', 'a|c']).str.get_dummies()
+       a  b  c
+    0  1  1  0
+    1  1  0  0
+    2  1  0  1
+
+    >>> pd.Series(['a|b', np.nan, 'a|c']).str.get_dummies()
+       a  b  c
+    0  1  1  0
+    1  0  0  0
+    2  1  0  1
+
+    See also ``pd.get_dummies``.
+
+    """
+    # TODO remove this hack?
+    arr = arr.fillna('')
+    try:
+        arr = sep + arr + sep
+    except TypeError:
+        arr = sep + arr.astype(str) + sep
+
+    tags = set()
+    for ts in arr.str.split(sep):
+        tags.update(ts)
+    tags = sorted(tags - set([""]))
+
+    dummies = np.empty((len(arr), len(tags)), dtype=int)
+
+    for i, t in enumerate(tags):
+        pat = sep + t + sep
+        dummies[:, i] = lib.map_infer(arr.values, lambda x: pat in x)
+    return DataFrame(dummies, arr.index, tags)
+
 
 def str_join(arr, sep):
     """
@@ -843,7 +882,7 @@ def contains(self, pat, case=True, flags=0, na=np.nan, regex=True):
         result = str_contains(self.series, pat, case=case, flags=flags,
                               na=na, regex=regex)
         return self._wrap_result(result)
-            
+
     @copy(str_replace)
     def replace(self, pat, repl, n=-1, case=True, flags=0):
         result = str_replace(self.series, pat, repl, n=n, case=case,
@@ -899,6 +938,11 @@ def rstrip(self, to_strip=None):
         result = str_rstrip(self.series, to_strip)
         return self._wrap_result(result)
 
+    @copy(str_get_dummies)
+    def get_dummies(self, sep='|'):
+        result = str_get_dummies(self.series, sep)
+        return self._wrap_result(result)
+
     count = _pat_wrapper(str_count, flags=True)
     startswith = _pat_wrapper(str_startswith, na=True)
     endswith = _pat_wrapper(str_endswith, na=True)
diff --git a/vb_suite/strings.py b/vb_suite/strings.py
--- a/vb_suite/strings.py
+++ b/vb_suite/strings.py
@@ -45,6 +45,11 @@ def make_series(letters, strlen, size):
 strings_rstrip = Benchmark("many.str.rstrip('matchthis')", setup)
 strings_get = Benchmark("many.str.get(0)", setup)
 
+setup = setup + """
+make_series(string.uppercase, strlen=10, size=10000).str.join('|')
+"""
+strings_get_dummies = Benchmark("s.str.get_dummies('|')", setup)
+
 setup = common_setup + """
 import pandas.util.testing as testing
 ser = pd.Series(testing.makeUnicodeIndex())


Test patch:
diff --git a/pandas/tests/test_strings.py b/pandas/tests/test_strings.py
--- a/pandas/tests/test_strings.py
+++ b/pandas/tests/test_strings.py
@@ -366,7 +366,6 @@ def test_replace(self):
         result = values.str.replace("(?<=\w),(?=\w)", ", ", flags=re.UNICODE)
         tm.assert_series_equal(result, exp)
 
-
     def test_repeat(self):
         values = Series(['a', 'b', NA, 'c', NA, 'd'])
 
@@ -465,7 +464,7 @@ def test_extract(self):
         # Contains tests like those in test_match and some others.
 
         values = Series(['fooBAD__barBAD', NA, 'foo'])
-        er = [NA, NA] # empty row
+        er = [NA, NA]  # empty row
 
         result = values.str.extract('.*(BAD[_]+).*(BAD)')
         exp = DataFrame([['BAD__', 'BAD'], er, er])
@@ -549,6 +548,19 @@ def test_extract(self):
         exp = DataFrame([['A', '1'], ['B', '2'], ['C', NA]], columns=['letter', 'number'])
         tm.assert_frame_equal(result, exp)
 
+    def test_get_dummies(self):
+        s = Series(['a|b', 'a|c', np.nan])
+        result = s.str.get_dummies('|')
+        expected = DataFrame([[1, 1, 0], [1, 0, 1], [0, 0, 0]],
+                             columns=list('abc'))
+        tm.assert_frame_equal(result, expected)
+
+        s = Series(['a;b', 'a', 7])
+        result = s.str.get_dummies(';')
+        expected = DataFrame([[0, 1, 1], [0, 1, 0], [1, 0, 0]],
+                             columns=list('7ab'))
+        tm.assert_frame_equal(result, expected)
+
     def test_join(self):
         values = Series(['a_b_c', 'c_d_e', np.nan, 'f_g_h'])
         result = values.str.split('_').str.join('_')