Repository: pandas-dev/pandas

Issue:
Yahoo options DataReader can't parse underlying prices with commas
I ran into this problem retrieving option data for the SPX settlement index:

```
>>> options_object = web.Options('^spxpm', 'yahoo')
>>> options_object.get_call_data( expiry=datetime.date(2014, 12, 20) )
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/pollyp/anaconda/lib/python2.7/site-packages/pandas/io/data.py", line 785, in get_call_data
    return self._get_data_in_date_range(expiry, call=True, put=False)
  File "/Users/pollyp/anaconda/lib/python2.7/site-packages/pandas/io/data.py", line 1098, in _get_data_in_date_range
    frame = self._get_option_data(expiry=expiry_date, name=name)
  File "/Users/pollyp/anaconda/lib/python2.7/site-packages/pandas/io/data.py", line 717, in _get_option_data
    frames = self._get_option_frames_from_yahoo(expiry)
  File "/Users/pollyp/anaconda/lib/python2.7/site-packages/pandas/io/data.py", line 655, in _get_option_frames_from_yahoo
    option_frames = self._option_frames_from_url(url)
  File "/Users/pollyp/anaconda/lib/python2.7/site-packages/pandas/io/data.py", line 684, in _option_frames_from_url
    self.underlying_price, self.quote_time = self._get_underlying_price(url)
  File "/Users/pollyp/anaconda/lib/python2.7/site-packages/pandas/io/data.py", line 696, in _get_underlying_price
    .getchildren()[0].text)
ValueError: invalid literal for float(): 2,071.92
```

Here's my show_versions output:

```
>>> pd.show_versions()

INSTALLED VERSIONS
------------------
commit: None
python: 2.7.8.final.0
python-bits: 64
OS: Darwin
OS-release: 13.4.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: None

pandas: 0.15.1
nose: 1.3.4
Cython: 0.21
numpy: 1.9.0
scipy: 0.14.0
statsmodels: 0.5.0
IPython: 2.2.0
sphinx: 1.2.3
patsy: 0.3.0
dateutil: 2.2
pytz: 2014.7
bottleneck: None
tables: 3.1.1
numexpr: 2.3.1
matplotlib: 1.4.0
openpyxl: 1.8.5
xlrd: 0.9.3
xlwt: 0.7.5
xlsxwriter: 0.5.7
lxml: 3.4.0
bs4: 4.3.2
html5lib: 0.9999-dev
httplib2: None
apiclient: None
rpy2: None
sqlalchemy: 0.9.7
pymysql: None
psycopg2: None
```



Discussion:
pull-requests to fix are welcome!

Excellent! I'll get it in this weekend.

Experiencing the same issue also on the S&P. 

```
INSTALLED VERSIONS
------------------
commit: None
python: 2.7.8.final.0
python-bits: 64
OS: Linux
OS-release: 3.16.0-28-generic
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8

pandas: 0.15.2
nose: 1.3.4
Cython: None
numpy: 1.8.2
scipy: 0.14.0
statsmodels: None
IPython: 2.3.0
sphinx: 1.2.2
patsy: None
dateutil: 2.2
pytz: 2014.7
bottleneck: None
tables: None
numexpr: None
matplotlib: 1.4.2
openpyxl: None
xlrd: None
xlwt: 0.7.5
xlsxwriter: None
lxml: 3.3.6
bs4: 4.3.2
html5lib: 0.999
httplib2: 0.9
apiclient: None
rpy2: 2.4.3
sqlalchemy: None
pymysql: None
psycopg2: None
```


Resolving patch:
diff --git a/pandas/io/data.py b/pandas/io/data.py
--- a/pandas/io/data.py
+++ b/pandas/io/data.py
@@ -171,7 +171,15 @@ def _retry_read_url(url, retry_count, pause, name):
             # return 2 rows for the most recent business day
             if len(rs) > 2 and rs.index[-1] == rs.index[-2]:  # pragma: no cover
                 rs = rs[:-1]
-            return rs
+
+    #Get rid of unicode characters in index name.
+    try:
+        rs.index.name = rs.index.name.decode('unicode_escape').encode('ascii', 'ignore')
+    except AttributeError:
+        #Python 3 string has no decode method.
+        rs.index.name = rs.index.name.encode('ascii', 'ignore').decode()
+
+    return rs
 
     raise IOError("after %d tries, %s did not "
                   "return a 200 for url %r" % (retry_count, name, url))
@@ -686,7 +694,7 @@ def _option_frames_from_url(self, url):
 
         if not hasattr(self, 'underlying_price'):
             try:
-                self.underlying_price, self.quote_time = self._get_underlying_price(url)
+                self.underlying_price, self.quote_time = self._underlying_price_and_time_from_url(url)
             except IndexError:
                 self.underlying_price, self.quote_time = np.nan, np.nan
 
@@ -701,23 +709,38 @@ def _option_frames_from_url(self, url):
 
         return {'calls': calls, 'puts': puts}
 
-    def _get_underlying_price(self, url):
+    def _underlying_price_and_time_from_url(self, url):
         root = self._parse_url(url)
-        underlying_price = float(root.xpath('.//*[@class="time_rtq_ticker Fz-30 Fw-b"]')[0]\
-            .getchildren()[0].text)
+        underlying_price = self._underlying_price_from_root(root)
+        quote_time = self._quote_time_from_root(root)
+        return underlying_price, quote_time
+
+    @staticmethod
+    def _underlying_price_from_root(root):
+        underlying_price = root.xpath('.//*[@class="time_rtq_ticker Fz-30 Fw-b"]')[0]\
+            .getchildren()[0].text
+        underlying_price = underlying_price.replace(',', '') #GH11
 
+        try:
+            underlying_price = float(underlying_price)
+        except ValueError:
+            underlying_price = np.nan
+
+        return underlying_price
+
+    @staticmethod
+    def _quote_time_from_root(root):
         #Gets the time of the quote, note this is actually the time of the underlying price.
         try:
             quote_time_text = root.xpath('.//*[@class="time_rtq Fz-m"]')[0].getchildren()[1].getchildren()[0].text
             ##TODO: Enable timezone matching when strptime can match EST with %Z
             quote_time_text = quote_time_text.split(' ')[0]
             quote_time = dt.datetime.strptime(quote_time_text, "%I:%M%p")
-
             quote_time = quote_time.replace(year=CUR_YEAR, month=CUR_MONTH, day=CUR_DAY)
         except ValueError:
             quote_time = np.nan
 
-        return underlying_price, quote_time
+        return quote_time
 
     def _get_option_data(self, expiry, name):
         frame_name = '_frames' + self._expiry_to_string(expiry)


Test patch:
diff --git a/pandas/io/tests/test_data.py b/pandas/io/tests/test_data.py
--- a/pandas/io/tests/test_data.py
+++ b/pandas/io/tests/test_data.py
@@ -117,6 +117,7 @@ def test_get_multi2(self):
                 self.assertEqual((4, 3), result.shape)
                 assert_n_failed_equals_n_null_columns(w, result)
 
+    @network
     def test_dtypes(self):
         #GH3995, #GH8980
         data = web.get_data_google('F', start='JAN-01-10', end='JAN-27-13')
@@ -126,6 +127,13 @@ def test_dtypes(self):
         assert np.issubdtype(data.High.dtype, np.number)
         assert np.issubdtype(data.Volume.dtype, np.number)
 
+    @network
+    def test_unicode_date(self):
+        #GH8967
+        data = web.get_data_google('F', start='JAN-01-10', end='JAN-27-13')
+        self.assertEquals(data.index.name, 'Date')
+
+
 class TestYahoo(tm.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -366,24 +374,35 @@ def test_get_all_data_calls_only(self):
         self.assertTrue(len(data) > 1)
 
     @network
+    def test_get_underlying_price(self):
+        #GH7
+        try:
+            options_object = web.Options('^spxpm', 'yahoo')
+            url = options_object._yahoo_url_from_expiry(options_object.expiry_dates[0])
+            root = options_object._parse_url(url)
+            quote_price = options_object._underlying_price_from_root(root)
+        except RemoteDataError as e:
+            raise nose.SkipTest(e)
+        self.assert_(isinstance(quote_price, float))
+
     def test_sample_page_price_quote_time1(self):
         #Tests the weekend quote time format
-        price, quote_time = self.aapl._get_underlying_price(self.html1)
-        self.assertIsInstance(price, (int, float, complex))
-        self.assertIsInstance(quote_time, (datetime, Timestamp))
+        price, quote_time = self.aapl._underlying_price_and_time_from_url(self.html1)
+        self.assert_(isinstance(price, (int, float, complex)))
+        self.assert_(isinstance(quote_time, (datetime, Timestamp)))
 
     def test_chop(self):
         #regression test for #7625
         self.aapl.chop_data(self.data1, above_below=2, underlying_price=np.nan)
         chopped = self.aapl.chop_data(self.data1, above_below=2, underlying_price=100)
-        self.assertIsInstance(chopped, DataFrame)
+        self.assert_(isinstance(chopped, DataFrame))
         self.assertTrue(len(chopped) > 1)
 
     def test_chop_out_of_strike_range(self):
         #regression test for #7625
         self.aapl.chop_data(self.data1, above_below=2, underlying_price=np.nan)
         chopped = self.aapl.chop_data(self.data1, above_below=2, underlying_price=100000)
-        self.assertIsInstance(chopped, DataFrame)
+        self.assert_(isinstance(chopped, DataFrame))
         self.assertTrue(len(chopped) > 1)
 
 
@@ -391,9 +410,9 @@ def test_chop_out_of_strike_range(self):
     def test_sample_page_price_quote_time2(self):
         #Tests the EDT page format
         #regression test for #8741
-        price, quote_time = self.aapl._get_underlying_price(self.html2)
-        self.assertIsInstance(price, (int, float, complex))
-        self.assertIsInstance(quote_time, (datetime, Timestamp))
+        price, quote_time = self.aapl._underlying_price_and_time_from_url(self.html2)
+        self.assert_(isinstance(price, (int, float, complex)))
+        self.assert_(isinstance(quote_time, (datetime, Timestamp)))
 
     @network
     def test_sample_page_chg_float(self):