Repository: pandas-dev/pandas

Issue:
BUG: impossible to select string with special character from HDFStore via query
http://stackoverflow.com/questions/21769406/pandas-and-hdf5-querying-a-table-string-containing-character

```
>>> from pandas import HDFStore, DataFrame
>>> df = DataFrame({'a': ['a', 'a', 'c', 'b', 'test & test', 'c' , 'b', 'e'], 
                   'b': [1, 2, 3, 4, 5, 6, 7, 8]})
>>> store = HDFStore('test.h5')
>>> store.append('test', df, format='table', data_columns=True)
>>> df[df.a == 'test & test']
     a              b
4    test & test    5
>>> store.select('test', 'a="test & test"')
```



Discussion:

Resolving patch:
diff --git a/pandas/computation/expr.py b/pandas/computation/expr.py
--- a/pandas/computation/expr.py
+++ b/pandas/computation/expr.py
@@ -252,7 +252,19 @@ def _replace_booleans(source):
     """Replace ``&`` with ``and`` and ``|`` with ``or`` so that bitwise
     precedence is changed to boolean precedence.
     """
-    return source.replace('|', ' or ').replace('&', ' and ')
+    res = []
+    g = tokenize.generate_tokens(StringIO(source).readline)
+    for toknum, tokval, _, _, _ in g:
+        if toknum == tokenize.OP:
+            if tokval == '&':
+                res.append((tokenize.NAME, 'and'))
+            elif tokval == '|':
+                res.append((tokenize.NAME, 'or'))
+            else:
+                res.append((toknum, tokval))
+        else:
+            res.append((toknum, tokval))
+    return tokenize.untokenize(res)
 
 
 def _replace_locals(source, local_symbol='@'):


Test patch:
diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py
--- a/pandas/io/tests/test_pytables.py
+++ b/pandas/io/tests/test_pytables.py
@@ -2199,7 +2199,7 @@ def test_remove_startstop(self):
         # GH #4835 and #6177
 
         with ensure_clean_store(self.path) as store:
-            
+
             wp = tm.makePanel()
 
             # start
@@ -2246,7 +2246,7 @@ def test_remove_startstop(self):
             result = store.select('wp6')
             expected = wp.reindex(major_axis=wp.major_axis)
             assert_panel_equal(result, expected)
-            
+
             # with where
             date = wp.major_axis.take(np.arange(0,30,3))
             crit = Term('major_axis=date')
@@ -2256,7 +2256,7 @@ def test_remove_startstop(self):
             result = store.select('wp7')
             expected = wp.reindex(major_axis=wp.major_axis-wp.major_axis[np.arange(0,20,3)])
             assert_panel_equal(result, expected)
-            
+
 
     def test_remove_crit(self):
 
@@ -4174,6 +4174,14 @@ def test_append_with_diff_col_name_types_raises_value_error(self):
                 with tm.assertRaises(ValueError):
                     store.append(name, d)
 
+    def test_query_with_nested_special_character(self):
+        df = DataFrame({'a': ['a', 'a', 'c', 'b', 'test & test', 'c' , 'b', 'e'],
+                        'b': [1, 2, 3, 4, 5, 6, 7, 8]})
+        expected = df[df.a == 'test & test']
+        with ensure_clean_store(self.path) as store:
+            store.append('test', df, format='table', data_columns=True)
+            result = store.select('test', 'a = "test & test"')
+        tm.assert_frame_equal(expected, result)
 
 def _test_sort(obj):
     if isinstance(obj, DataFrame):
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -12875,6 +12875,19 @@ def test_query_with_nested_string(self):
         for parser, engine in product(PARSERS, ENGINES):
             yield self.check_query_with_nested_strings, parser, engine
 
+    def check_query_with_nested_special_character(self, parser, engine):
+        skip_if_no_pandas_parser(parser)
+        tm.skip_if_no_ne(engine)
+        df = DataFrame({'a': ['a', 'b', 'test & test'],
+                        'b': [1, 2, 3]})
+        res = df.query('a == "test & test"', parser=parser, engine=engine)
+        expec = df[df.a == 'test & test']
+        tm.assert_frame_equal(res, expec)
+
+    def test_query_with_nested_special_character(self):
+        for parser, engine in product(PARSERS, ENGINES):
+            yield self.check_query_with_nested_special_character, parser, engine
+
     def check_query_lex_compare_strings(self, parser, engine):
         tm.skip_if_no_ne(engine=engine)
         import operator as opr