Repository: eWaterCycle/era5cli

Issue:
Unexpected behavior for --statistics flag
When downloading hourly data you can select the `--statistics` flag without using the `--ensemble` flag. This will result in downloading the reanalysis data instead and will ignore the `--statistics` flag. The `--statistics` flag without `--ensemble` should at least give a warning or an error, as adding this flag is meaningless when not downloading ensemble data.

Comments:

Patch:
diff --git a/era5cli/cli.py b/era5cli/cli.py
index a13ce9c..4dc2de2 100644
--- a/era5cli/cli.py
+++ b/era5cli/cli.py
@@ -5,6 +5,8 @@ import argparse
 import textwrap
 import sys
 
+from datetime import datetime
+
 import era5cli.inputref as ref
 import era5cli.info as einfo
 import era5cli.fetch as efetch
@@ -127,6 +129,17 @@ def _build_parser():
                              ''')
     )
 
+    common.add_argument(
+        "--prelimbe", action="store_true", default=False,
+        help=textwrap.dedent('''\
+                             Whether to download the preliminary back extension
+                             (1950-1978). Providing the
+                             "--prelimbe" argument downloads data from
+                             the preliminary back extension.
+
+                             ''')
+    )
+
     mnth = argparse.ArgumentParser(add_help=False)
 
     mnth.add_argument(
@@ -271,6 +284,32 @@ def _run_info(args):
         return True
 
 
+def _construct_year_list(args):
+    if not args.endyear:
+        endyear = args.startyear
+    else:
+        endyear = args.endyear
+
+    # check whether correct years have been entered
+    for year in (args.startyear, endyear):
+        if args.prelimbe:
+            assert 1950 <= year <= 1978, (
+                'year should be between 1950 and 1978'
+            )
+        else:
+            assert 1979 <= year <= datetime.now().year, (
+                'year should be between 1979 and present'
+            )
+
+    assert endyear >= args.startyear, (
+        'endyear should be >= startyear or None')
+
+    # make list of years to be downloaded
+    years = list(range(args.startyear, endyear + 1))
+
+    return years
+
+
 def _set_period_args(args):
     # set subroutine specific arguments for monthly and hourly fetch
     if args.command == "monthly":
@@ -288,6 +327,11 @@ def _set_period_args(args):
     elif args.command == "hourly":
         synoptic = None
         statistics = args.statistics
+        if statistics:
+            assert args.ensemble, (
+                "Statistics can only be computed over an ensemble, "
+                "add --ensemble or remove --statistics."
+                )
         days = args.days
         hours = args.hours
     else:
@@ -305,30 +349,26 @@ def _execute(args):
 
     # the fetching subroutines
     else:
-        # make list of years to be downloaded
-        if not args.endyear:
-            years = [args.startyear]
-        else:
-            assert (args.endyear >= args.startyear), (
-                'endyear should be >= startyear or None')
-            years = list(range(args.startyear, args.endyear + 1))
-
+        years = _construct_year_list(args)
         synoptic, statistics, days, hours = _set_period_args(args)
         # try to build and send download request
-        era5 = efetch.Fetch(years,
-                            months=args.months,
-                            days=days,
-                            hours=hours,
-                            variables=args.variables,
-                            outputformat=args.format,
-                            outputprefix=args.outputprefix,
-                            period=args.command,
-                            ensemble=args.ensemble,
-                            synoptic=synoptic,
-                            statistics=statistics,
-                            pressurelevels=args.levels,
-                            threads=args.threads,
-                            merge=args.merge)
+        era5 = efetch.Fetch(
+            years,
+            months=args.months,
+            days=days,
+            hours=hours,
+            variables=args.variables,
+            outputformat=args.format,
+            outputprefix=args.outputprefix,
+            period=args.command,
+            ensemble=args.ensemble,
+            synoptic=synoptic,
+            statistics=statistics,
+            pressurelevels=args.levels,
+            threads=args.threads,
+            merge=args.merge,
+            prelimbe=args.prelimbe,
+        )
         era5.fetch(dryrun=args.dryrun)
         return True
 
diff --git a/era5cli/fetch.py b/era5cli/fetch.py
index 6333e79..d67b1e6 100644
--- a/era5cli/fetch.py
+++ b/era5cli/fetch.py
@@ -57,13 +57,15 @@ class Fetch:
             Indicating if files should be downloaded. By default
             files will be downloaded. For a dryrun the cdsapi request will
             be written to stdout.
+        prelimbe: bool
+            Whether to download the preliminary back extension (1950-1978).
     """
 
     def __init__(self, years: list, months: list, days: list,
                  hours: list, variables: list, outputformat: str,
                  outputprefix: str, period: str, ensemble: bool,
                  statistics=None, synoptic=None, pressurelevels=None,
-                 merge=False, threads=None):
+                 merge=False, threads=None, prelimbe=False):
         """Initialization of Fetch class."""
         self.months = era5cli.utils._zpad_months(months)
         """list(str): List of zero-padded strings of months
@@ -107,6 +109,9 @@ class Fetch:
         """bool: Whether to get monthly averaged by hour of day
         (synoptic=True) or monthly means of daily means
         (synoptic=False)."""
+        self.prelimbe = prelimbe
+        """bool: Whether to select from the ERA5 preliminary back
+        extension which supports years from 1950 to 1978"""
 
     def fetch(self, dryrun=False):
         """Split calls and fetch results.
@@ -195,17 +200,25 @@ class Fetch:
         elif not self.ensemble:
             producttype += "reanalysis"
 
-        if self.period == "monthly":
+        if self.period == "monthly" and not self.prelimbe:
             producttype = "monthly_averaged_" + producttype
             if self.synoptic:
                 producttype += "_by_hour_of_day"
-        elif self.period == "hourly":
-            if self.ensemble and self.statistics:
-                producttype = [
-                    "ensemble_members",
-                    "ensemble_mean",
-                    "ensemble_spread",
-                ]
+        elif self.period == "monthly" and self.prelimbe:
+            if self.ensemble:
+                producttype = "members-"
+            elif not self.ensemble:
+                producttype = "reanalysis-"
+            if self.synoptic:
+                producttype += "synoptic-monthly-means"
+            elif not self.synoptic:
+                producttype += "monthly-means-of-daily-means"
+        elif self.period == "hourly" and self.ensemble and self.statistics:
+            producttype = [
+                "ensemble_members",
+                "ensemble_mean",
+                "ensemble_spread",
+            ]
 
         return producttype
 
@@ -253,6 +266,9 @@ class Fetch:
             if self.days:
                 request["day"] = self.days
 
+        if self.prelimbe:
+            name += "-preliminary-back-extension"
+
         return(name, request)
 
     def _exit(self):


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 2be478d..13c1cda 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -47,6 +47,14 @@ def test_period_args():
     # Period_args consists of (synoptic, statistics, days, hours)
     assert period_args == (True, None, None, [4, 7])
 
+    argv = ['monthly', '--startyear', '2008',
+            '--variables', 'total_precipitation',
+            '--synoptic', '--ensemble']
+    args = cli._parse_args(argv)
+    period_args = cli._set_period_args(args)
+    # Period_args consists of (synoptic, statistics, days, hours)
+    assert period_args == (True, None, None, range(0, 24))
+
     # test whether the info option does not end up in _set_period_args
     argv = ['info', '2Dvars']
     args = cli._parse_args(argv)
@@ -71,6 +79,22 @@ def test_main_fetch(fetch):
     with pytest.raises(AssertionError):
         assert cli._execute(args)
 
+    # should give an AssertionError if years are out of bounds
+    argv = ['hourly', '--startyear', '1950',
+            '--variables', 'total_precipitation', '--statistics',
+            '--endyear', '2007', '--ensemble']
+    args = cli._parse_args(argv)
+    with pytest.raises(AssertionError):
+        assert cli._execute(args)
+
+    # should give an AssertionError if years are out of bounds
+    argv = ['hourly', '--startyear', '1950',
+            '--variables', 'total_precipitation', '--statistics',
+            '--endyear', '2007', '--ensemble', '--prelimbe']
+    args = cli._parse_args(argv)
+    with pytest.raises(AssertionError):
+        assert cli._execute(args)
+
     # monthly call without endyear
     argv = ['monthly', '--startyear', '2008',
             '--variables', 'total_precipitation', '--synoptic',
diff --git a/tests/test_fetch.py b/tests/test_fetch.py
index 1690c26..cff9ff8 100644
--- a/tests/test_fetch.py
+++ b/tests/test_fetch.py
@@ -9,7 +9,8 @@ def initialize(outputformat='netcdf', merge=False, statistics=None,
                synoptic=None, ensemble=True, pressurelevels=None,
                threads=2, period='hourly', variables=['total_precipitation'],
                years=[2008, 2009], months=list(range(1, 13)),
-               days=list(range(1, 32)), hours=list(range(0, 24))):
+               days=list(range(1, 32)), hours=list(range(0, 24)),
+               prelimbe=False):
     """Initializer of the class."""
     era5 = fetch.Fetch(years=years,
                        months=months,
@@ -24,7 +25,8 @@ def initialize(outputformat='netcdf', merge=False, statistics=None,
                        synoptic=synoptic,
                        pressurelevels=pressurelevels,
                        merge=merge,
-                       threads=threads)
+                       threads=threads,
+                       prelimbe=prelimbe)
     return era5
 
 
@@ -43,7 +45,8 @@ def test_init():
                        synoptic=None,
                        pressurelevels=None,
                        merge=False,
-                       threads=2)
+                       threads=2,
+                       prelimbe=False)
 
     valid_months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
                     '11', '12']
@@ -71,6 +74,7 @@ def test_init():
     assert era5.pressure_levels is None
     assert not era5.merge
     assert era5.threads == 2
+    assert not era5.prelimbe
 
     # initializing hourly variable with days=None should result in ValueError
     with pytest.raises(TypeError):
@@ -247,10 +251,24 @@ def test_product_type():
     producttype = era5._product_type()
     assert producttype == 'monthly_averaged_reanalysis'
 
+    era5.prelimbe = True
+    producttype = era5._product_type()
+    assert producttype == 'reanalysis-monthly-means-of-daily-means'
+
+    era5.prelimbe = False
     era5.synoptic = True
     producttype = era5._product_type()
     assert producttype == 'monthly_averaged_reanalysis_by_hour_of_day'
 
+    era5.prelimbe = True
+    producttype = era5._product_type()
+    assert producttype == 'reanalysis-synoptic-monthly-means'
+
+    era5.ensemble = True
+    producttype = era5._product_type()
+    assert producttype == 'members-synoptic-monthly-means'
+
+    era5.prelimbe = False
     era5.ensemble = False
     era5.statistics = True
     producttype = era5._product_type()
@@ -278,7 +296,6 @@ def test_build_request():
                     '12:00', '13:00', '14:00', '15:00', '16:00', '17:00',
                     '18:00', '19:00', '20:00', '21:00', '22:00', '23:00'],
            'format': 'netcdf'}
-    print(request['day'])
     assert request == req
 
     # monthly data
@@ -298,6 +315,28 @@ def test_build_request():
            'format': 'netcdf'}
     assert request == req
 
+    # preliminary back extension
+    era5 = initialize(period='monthly',
+                      variables=['total_precipitation'],
+                      years=[1970],
+                      prelimbe=True)
+    (name, request) = era5._build_request('total_precipitation', [1970])
+    print(request)
+    assert name == (
+        "reanalysis-era5-single-levels-monthly"
+        "-means-preliminary-back-extension"
+    )
+    req = {'variable': 'total_precipitation', 'year': [1970],
+           'product_type': 'members-monthly-means-of-daily-means',
+           'month': ['01', '02', '03', '04', '05', '06',
+                     '07', '08', '09', '10', '11', '12'],
+           'time': ['00:00', '01:00', '02:00', '03:00', '04:00', '05:00',
+                    '06:00', '07:00', '08:00', '09:00', '10:00', '11:00',
+                    '12:00', '13:00', '14:00', '15:00', '16:00', '17:00',
+                    '18:00', '19:00', '20:00', '21:00', '22:00', '23:00'],
+           'format': 'netcdf'}
+    assert request == req
+
     # requesting 3d variable with pressurelevels=None should give a ValueError
     era5 = initialize(variables=['temperature'], pressurelevels=None)
     with pytest.raises(ValueError):