Repository: epogrebnyak/weo-reader

Issue:
doesn't work for 2020-October database
weo package doesn't seem to work for 2020-October database.
running this code:

```
from weo import download
from weo import WEO

download("2020-Oct", path='weo.csv', overwrite=True)
w = WEO("weo.csv")
```

gives the following error:
---------------------------------------------------------------------------
ParserError                               Traceback (most recent call last)
<ipython-input-14-90829dcd1cb0> in <module>
----> 1 w = WEO("weo.csv")

~/.local/lib/python3.7/site-packages/weo/dataframe.py in __init__(self, filename)
    111 
    112     def __init__(self, filename):
--> 113         self.df, _ = read_csv(filename)
    114 
    115     @property

~/.local/lib/python3.7/site-packages/weo/dataframe.py in read_csv(filename)
     31 
     32 def read_csv(filename):
---> 33     df = pd.read_csv(filename, delimiter="\t", encoding="iso-8859-1")
     34     ix = df["Country"].isna()
     35     return df[~ix], df[ix]

~/.local/lib/python3.7/site-packages/pandas/io/parsers.py in read_csv(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)
    686     )
    687 
--> 688     return _read(filepath_or_buffer, kwds)
    689 
    690 

~/.local/lib/python3.7/site-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds)
    458 
    459     try:
--> 460         data = parser.read(nrows)
    461     finally:
    462         parser.close()

~/.local/lib/python3.7/site-packages/pandas/io/parsers.py in read(self, nrows)
   1196     def read(self, nrows=None):
   1197         nrows = _validate_integer("nrows", nrows)
-> 1198         ret = self._engine.read(nrows)
   1199 
   1200         # May alter columns / col_dict

~/.local/lib/python3.7/site-packages/pandas/io/parsers.py in read(self, nrows)
   2155     def read(self, nrows=None):
   2156         try:
-> 2157             data = self._reader.read(nrows)
   2158         except StopIteration:
   2159             if self._first_chunk:

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.read()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_low_memory()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_rows()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._tokenize_rows()

pandas/_libs/parsers.pyx in pandas._libs.parsers.raise_parser_error()

ParserError: Error tokenizing data. C error: Expected 1 fields in line 48, saw 2

Comments:

Patch:
diff --git a/weo/dataframe.py b/weo/dataframe.py
index 2715367..5fed1c7 100644
--- a/weo/dataframe.py
+++ b/weo/dataframe.py
@@ -29,8 +29,11 @@ def convert(x):
         return np.nan
 
 
-def read_csv(filename):
+def read_csv(filename):  # October 2020 and later files use UTF-16 LE encoding
     df = pd.read_csv(filename, delimiter="\t", encoding="iso-8859-1")
+    if df.isnull().iloc[0, 0]:
+        df = pd.read_csv(filename, delimiter="\t", encoding="UTF-16 LE")
+        df.dropna(how="all", axis=1, inplace=True)
     ix = df["Country"].isna()
     return df[~ix], df[ix]
 
@@ -58,7 +61,7 @@ def accept_year(func):  # FIXME: make accept a country
         if arg:
             year = arg[0]
         if year:
-            ts = df[str(year)].transpose().iloc[:, 0]
+            ts = df.transpose()[str(year)]
             return ts
         else:
             return df
@@ -325,7 +328,7 @@ class WEO:
         if year is None:
             return _df
         else:
-            _df = _df[str(year)].transpose()
+            _df = _df.transpose()[str(year)]
             _df["Description"] = _df.index.map(lambda c: " - ".join(self.from_code(c)))
             return _df
 
diff --git a/weo/download.py b/weo/download.py
index 471a55e..c854b22 100644
--- a/weo/download.py
+++ b/weo/download.py
@@ -1,11 +1,11 @@
 """Download dataset from IMF WEO website.
 
   from weo import download
-  download("2019-Oct', 'weo.csv')       
+  download("2020-Oct', 'weo.csv')       
            
 Equivalent to:
 
-  curl -o weo.csv https://www.imf.org/external/pubs/ft/weo/2019/02/weodata/WEOOct2019all.xls
+  curl -o weo.csv https://www.imf.org/-/media/Files/Publications/WEO/WEO-Database/2020/02/WEOOct2020all.xls
 
 """
 
@@ -48,15 +48,23 @@ def make_url(r, prefix):
     Data in other formats goes back to 2000.
 
     Landing page with URLs:
-    https://www.imf.org/external/pubs/ft/weo/2011/02/weodata/download.aspx
+    https://www.imf.org/en/Publications/SPROLLs/world-economic-outlook-databases
     """
+
+    base_url = "https://www.imf.org/-/media/Files/Publications/WEO/WEO-Database"
+
     year = r.year
     month = r.month_string()
     period_marker = r.period_string()
-    return (
-        "https://www.imf.org/external/pubs/ft/weo/"
-        f"{year}/{period_marker}"
-        f"/weodata/WEO{month}{year}{prefix}.xls"
+
+    if year > 2020 or (year == 2020 and month == "Oct"):  # New URL format
+        return (
+            f"{base_url}/{year}/{period_marker}"
+            f"/WEO{month}{year}{prefix}.xls"
+        )
+    return (  # Old URL format
+        f"{base_url}/{year}"
+        f"/WEO{month}{year}{prefix}.xls"
     )
 
 


Test patch:
diff --git a/tests/test_weo_dates.py b/tests/test_weo_dates.py
index 9cc2679..acd740f 100644
--- a/tests/test_weo_dates.py
+++ b/tests/test_weo_dates.py
@@ -4,7 +4,7 @@ from weo import dates, all_dates
 def test_dates():
     assert dates(2007) == ["2007-Oct"]
     assert dates(2011) == ["2011-Apr", "2011-Sep"]
-    assert dates(2020) == ["2020-Apr"]
+    assert dates(2020) == ["2020-Apr", "2020-Oct"]
 
 
 def test_all_dates():
diff --git a/tests/test_weo_download.py b/tests/test_weo_download.py
index f5ac762..5af1f78 100644
--- a/tests/test_weo_download.py
+++ b/tests/test_weo_download.py
@@ -16,14 +16,14 @@ def test_from_date_fails():
 def test_make_countries_url():
     assert (
         make_url_countries(from_date("2020-04"))
-        == "https://www.imf.org/external/pubs/ft/weo/2020/01/weodata/WEOApr2020all.xls"
+        == "https://www.imf.org/-/media/Files/Publications/WEO/WEO-Database/2020/WEOApr2020all.xls"
     )
 
 
 def test_url_september():
     assert (
         make_url_countries(from_date("2011-Sep"))
-        == "https://www.imf.org/external/pubs/ft/weo/2011/02/weodata/WEOSep2011all.xls"
+        == "https://www.imf.org/-/media/Files/Publications/WEO/WEO-Database/2011/WEOSep2011all.xls"
     )