Repository: mlouielu/twstock

Issue:
2017 12/18 大樹(6469) Error
2017 12/18 大樹(6469) ，當天無交易量，證交所的資料為 '--'
程式會出現 Error。

希望能自動拿前一天的收盤價取代表示。

Comments:

Patch:
diff --git a/twstock/stock.py b/twstock/stock.py
index 842c4ff..ba2f706 100644
--- a/twstock/stock.py
+++ b/twstock/stock.py
@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 
 import datetime
-import json
 import urllib.parse
 from collections import namedtuple
-import sys
+try:
+    from json.decoder import JSONDecodeError
+except ImportError:
+    JSONDecodeError = ValueError
 
 import requests
 
@@ -43,23 +45,19 @@ class TWSEFetcher(BaseFetcher):
     def __init__(self):
         pass
 
-    def fetch(self, year: int, month: int, sid: str, retry=5):
+    def fetch(self, year: int, month: int, sid: str, retry: int=5):
         params = {'date': '%d%02d01' % (year, month), 'stockNo': sid}
-        r = requests.get(self.REPORT_URL, params=params)
-        if sys.version_info < (3, 5):
+        for retry_i in range(retry):
+            r = requests.get(self.REPORT_URL, params=params)
             try:
                 data = r.json()
-            except ValueError:
-                if retry:
-                    return self.fetch(year, month, sid, retry - 1)
-                data = {'stat': '', 'data': []}
+            except JSONDecodeError:
+                continue
+            else:
+                break
         else:
-            try:
-                data = r.json()
-            except json.decoder.JSONDecodeError:
-                if retry:
-                    return self.fetch(year, month, sid, retry - 1)
-                data = {'stat': '', 'data': []}
+            # Fail in all retries
+            data = {'stat': '', 'data': []}
 
         if data['stat'] == 'OK':
             data['data'] = self.purify(data)
@@ -71,16 +69,17 @@ class TWSEFetcher(BaseFetcher):
         data[0] = datetime.datetime.strptime(self._convert_date(data[0]), '%Y/%m/%d')
         data[1] = int(data[1].replace(',', ''))
         data[2] = int(data[2].replace(',', ''))
-        data[3] = float(data[3].replace(',', ''))
-        data[4] = float(data[4].replace(',', ''))
-        data[5] = float(data[5].replace(',', ''))
-        data[6] = float(data[6].replace(',', ''))
-        data[7] = float(0.0 if data[7].replace(',', '') == 'X0.00' else data[7].replace(',', ''))  # +/-/X表示漲/跌/不比價
+        data[3] = None if data[3] == '--' else float(data[3].replace(',', ''))
+        data[4] = None if data[4] == '--' else float(data[4].replace(',', ''))
+        data[5] = None if data[5] == '--' else float(data[5].replace(',', ''))
+        data[6] = None if data[6] == '--' else float(data[6].replace(',', ''))
+        # +/-/X表示漲/跌/不比價
+        data[7] = float(0.0 if data[7].replace(',', '') == 'X0.00' else data[7].replace(',', ''))
         data[8] = int(data[8].replace(',', ''))
         return DATATUPLE(*data)
 
     def purify(self, original_data):
-        return [self._make_datatuple(d) for d in original_data['data'] if d[3] != '--']
+        return [self._make_datatuple(d) for d in original_data['data']]
 
 
 class TPEXFetcher(BaseFetcher):
@@ -90,10 +89,19 @@ class TPEXFetcher(BaseFetcher):
     def __init__(self):
         pass
 
-    def fetch(self, year: int, month: int, sid: str):
+    def fetch(self, year: int, month: int, sid: str, retry: int=5):
         params = {'d': '%d/%d' % (year - 1911, month), 'stkno': sid}
-        r = requests.get(self.REPORT_URL, params=params)
-        data = r.json()
+        for retry_i in range(retry):
+            r = requests.get(self.REPORT_URL, params=params)
+            try:
+                data = r.json()
+            except JSONDecodeError:
+                continue
+            else:
+                break
+        else:
+            # Fail in all retries
+            data = {'aaData': []}
 
         data['data'] = []
         if data['aaData']:
@@ -108,10 +116,10 @@ class TPEXFetcher(BaseFetcher):
         data[0] = datetime.datetime.strptime(self._convert_date(data[0]), '%Y/%m/%d')
         data[1] = int(data[1].replace(',', '')) * 1000
         data[2] = int(data[2].replace(',', '')) * 1000
-        data[3] = float(data[3].replace(',', ''))
-        data[4] = float(data[4].replace(',', ''))
-        data[5] = float(data[5].replace(',', ''))
-        data[6] = float(data[6].replace(',', ''))
+        data[3] = None if data[3] == '--' else float(data[3].replace(',', ''))
+        data[4] = None if data[4] == '--' else float(data[4].replace(',', ''))
+        data[5] = None if data[5] == '--' else float(data[5].replace(',', ''))
+        data[6] = None if data[6] == '--' else float(data[6].replace(',', ''))
         data[7] = float(data[7].replace(',', ''))
         data[8] = int(data[8].replace(',', ''))
         return DATATUPLE(*data)


Test patch:
diff --git a/test/test_stock.py b/test/test_stock.py
index ea4c7b6..92fbbd2 100644
--- a/test/test_stock.py
+++ b/test/test_stock.py
@@ -23,6 +23,20 @@ class FetcherTest(object):
         self.assertEqual(dt.change, 2.0)
         self.assertEqual(dt.transaction, 15718)
 
+    def test_make_datatuple_without_prices(self):
+        data = ['106/05/02', '45,851,963', '9,053,856,108', '--',
+                '--', '--', '--', ' 0.00', '15,718']
+        dt = self.fetcher._make_datatuple(data)
+        self.assertEqual(dt.date, datetime.datetime(2017, 5, 2))
+        self.assertEqual(dt.capacity, 45851963)
+        self.assertEqual(dt.turnover, 9053856108)
+        self.assertEqual(dt.open, None)
+        self.assertEqual(dt.high, None)
+        self.assertEqual(dt.low, None)
+        self.assertEqual(dt.close, None)
+        self.assertEqual(dt.change, 0.0)
+        self.assertEqual(dt.transaction, 15718)
+
 
 class TWSEFetcerTest(unittest.TestCase, FetcherTest):
     fetcher = stock.TWSEFetcher()
@@ -33,7 +47,7 @@ class TPEXFetcherTest(unittest.TestCase, FetcherTest):
 
     def test_make_datatuple(self):
         data = ['106/05/02', '45,851', '9,053,856', '198.50',
-                '199.00', '195.50', '196.50', '+2.00', '15,718']
+                '199.00', '195.50', '196.50', '2.00', '15,718']
         dt = self.fetcher._make_datatuple(data)
         self.assertEqual(dt.date, datetime.datetime(2017, 5, 2))
         self.assertEqual(dt.capacity, 45851000)
@@ -45,6 +59,20 @@ class TPEXFetcherTest(unittest.TestCase, FetcherTest):
         self.assertEqual(dt.change, 2.0)
         self.assertEqual(dt.transaction, 15718)
 
+    def test_make_datatuple_without_prices(self):
+        data = ['106/05/02', '45,851', '9,053,856', '--',
+                '--', '--', '--', '0.00', '15,718']
+        dt = self.fetcher._make_datatuple(data)
+        self.assertEqual(dt.date, datetime.datetime(2017, 5, 2))
+        self.assertEqual(dt.capacity, 45851000)
+        self.assertEqual(dt.turnover, 9053856000)
+        self.assertEqual(dt.open, None)
+        self.assertEqual(dt.high, None)
+        self.assertEqual(dt.low, None)
+        self.assertEqual(dt.close, None)
+        self.assertEqual(dt.change, 0.0)
+        self.assertEqual(dt.transaction, 15718)
+
 
 class StockTest(object):
     def test_fetch_31(self):