Repository: ESMValGroup/ESMValCore

Issue:
Add SUM operator for daily_statistics preprocessor function
This is useful to compute e.g. daily precipitation flux from hourly data.

Discussion:

Patch:
diff --git a/README.md b/README.md
index 8dc3f49e5..0b81db984 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # ESMValCore package
 
-[![Documentation Status](https://readthedocs.org/projects/esmvaltool/badge/?version=latest)](https://esmvaltool.readthedocs.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/esmvaltool/badge/?version=version2_development)](https://esmvaltool.readthedocs.io/en/version2_development/?badge=version2_development)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3387139.svg)](https://doi.org/10.5281/zenodo.3387139)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ESMValGroup?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![CircleCI](https://circleci.com/gh/ESMValGroup/ESMValCore.svg?style=svg)](https://circleci.com/gh/ESMValGroup/ESMValCore)
diff --git a/doc/esmvalcore/preprocessor.rst b/doc/esmvalcore/preprocessor.rst
index 102862b8b..43552aa94 100644
--- a/doc/esmvalcore/preprocessor.rst
+++ b/doc/esmvalcore/preprocessor.rst
@@ -687,7 +687,7 @@ This function produces statistics for each day in the dataset.
 
 Parameters:
     * operator: operation to apply. Accepted values are 'mean',
-      'median', 'std_dev', 'min' and 'max'. Default is 'mean'
+      'median', 'std_dev', 'min', 'max' and 'sum'. Default is 'mean'
 
 See also :func:`esmvalcore.preprocessor.daily_statistics`.
 
@@ -700,7 +700,7 @@ This function produces statistics for each month in the dataset.
 
 Parameters:
     * operator: operation to apply. Accepted values are 'mean',
-      'median', 'std_dev', 'min' and 'max'. Default is 'mean'
+      'median', 'std_dev', 'min', 'max' and 'sum'. Default is 'mean'
 
 See also :func:`esmvalcore.preprocessor.monthly_statistics`.
 
@@ -720,7 +720,7 @@ December and remove such biased initial datapoints.
 
 Parameters:
     * operator: operation to apply. Accepted values are 'mean',
-      'median', 'std_dev', 'min' and 'max'. Default is 'mean'
+      'median', 'std_dev', 'min', 'max' and 'sum'. Default is 'mean'
 
 See also :func:`esmvalcore.preprocessor.seasonal_mean`.
 
@@ -733,7 +733,7 @@ This function produces statistics for each year.
 
 Parameters:
     * operator: operation to apply. Accepted values are 'mean',
-      'median', 'std_dev', 'min' and 'max'. Default is 'mean'
+      'median', 'std_dev', 'min', 'max' and 'sum'. Default is 'mean'
 
 See also :func:`esmvalcore.preprocessor.annual_statistics`.
 
@@ -746,7 +746,7 @@ This function produces statistics for each decade.
 
 Parameters:
     * operator: operation to apply. Accepted values are 'mean',
-      'median', 'std_dev', 'min' and 'max'. Default is 'mean'
+      'median', 'std_dev', 'min', 'max' and 'sum'. Default is 'mean'
 
 See also :func:`esmvalcore.preprocessor.decadal_statistics`.
 
@@ -759,8 +759,8 @@ This function produces statistics for the whole dataset. It can produce scalars
 (if the full period is chosen) or daily, monthly or seasonal statics.
 
 Parameters:
-    * operator: operation to apply. Accepted values are 'mean', 'median', 'std_dev',
-      'min' and 'max'. Default is 'mean'
+    * operator: operation to apply. Accepted values are 'mean', 'median',
+      'std_dev', 'min', 'max' and 'sum'. Default is 'mean'
 
     * period: define the granularity of the statistics: get values for the
       full period, for each month or day of year.
@@ -924,8 +924,8 @@ named ``zonal_mean``, it can be used to apply several different operations in
 an zonal or meridional direction. This function takes two arguments:
 
 * ``coordinate``: Which direction to apply the operation: latitude or longitude
-* ``mean_type``: Which operation to apply: mean, std_dev, variance, median, min
-  or max
+* ``mean_type``: Which operation to apply: mean, std_dev, variance, median,
+  min, max or sum
 
 See also :func:`esmvalcore.preprocessor.zonal_means`.
 
diff --git a/esmvalcore/cmor/_fixes/cmip5/ec_earth.py b/esmvalcore/cmor/_fixes/cmip5/ec_earth.py
index 2f0f21f9c..7b233e624 100644
--- a/esmvalcore/cmor/_fixes/cmip5/ec_earth.py
+++ b/esmvalcore/cmor/_fixes/cmip5/ec_earth.py
@@ -2,7 +2,6 @@
 from dask import array as da
 
 from ..fix import Fix
-from ..shared import add_scalar_height_coord
 
 
 class Sic(Fix):
@@ -73,31 +72,3 @@ class Tos(Fix):
         """
         cube.data = da.ma.masked_equal(cube.core_data(), 273.15)
         return cube
-
-
-class Tas(Fix):
-    """Fixes for tas."""
-
-    def fix_metadata(self, cubes):
-        """
-        Fix potentially missing scalar dimension.
-
-        Parameters
-        ----------
-        cubes: iris CubeList
-            List of cubes to fix
-
-        Returns
-        -------
-        iris.cube.CubeList
-
-        """
-
-        for cube in cubes:
-            if not cube.coords(var_name='height'):
-                add_scalar_height_coord(cube)
-
-            if cube.coord('time').long_name is None:
-                cube.coord('time').long_name = 'time'
-
-        return cubes
diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py
index 6db338ff5..c03cbc528 100644
--- a/esmvalcore/preprocessor/_area.py
+++ b/esmvalcore/preprocessor/_area.py
@@ -13,7 +13,8 @@ import shapely
 import shapely.ops
 from dask import array as da
 
-from ._shared import get_iris_analysis_operation, guess_bounds
+from ._shared import (get_iris_analysis_operation, guess_bounds,
+                      operator_accept_weights)
 
 logger = logging.getLogger(__name__)
 
@@ -95,6 +96,7 @@ def zonal_means(cube, coordinate, mean_type):
     - 'mean' -> MEAN
     - 'median' -> MEDIAN
     - 'std_dev' -> STD_DEV
+    - 'sum' -> SUM
     - 'variance' -> VARIANCE
     - 'min' -> MIN
     - 'max' -> MAX
@@ -176,6 +178,8 @@ def area_statistics(cube, operator, fx_files=None):
     +------------+--------------------------------------------------+
     | `std_dev`  | Standard Deviation (not area weighted)           |
     +------------+--------------------------------------------------+
+    | `sum`      | Area weighted sum.                               |
+    +------------+--------------------------------------------------+
     | `variance` | Variance (not area weighted)                     |
     +------------+--------------------------------------------------+
     | `min`:     | Minimum value                                    |
@@ -188,7 +192,8 @@ def area_statistics(cube, operator, fx_files=None):
         cube: iris.cube.Cube
             Input cube.
         operator: str
-            The operation, options: mean, median, min, max, std_dev, variance
+            The operation, options: mean, median, min, max, std_dev, sum,
+            variance
         fx_files: dict
             dictionary of field:filename for the fx_files
 
@@ -226,7 +231,7 @@ def area_statistics(cube, operator, fx_files=None):
     # TODO: implement weighted stdev, median, s var when available in iris.
     # See iris issue: https://github.com/SciTools/iris/issues/3208
 
-    if operator == 'mean':
+    if operator_accept_weights(operator):
         return cube.collapsed(coord_names, operation, weights=grid_areas)
 
     # Many IRIS analysis functions do not accept weights arguments.
diff --git a/esmvalcore/preprocessor/_shared.py b/esmvalcore/preprocessor/_shared.py
index 385881ae1..73a7892e7 100644
--- a/esmvalcore/preprocessor/_shared.py
+++ b/esmvalcore/preprocessor/_shared.py
@@ -40,9 +40,9 @@ def get_iris_analysis_operation(operator):
     ------
     ValueError
         operator not in allowed operators list.
-        allowed operators: mean, median, std_dev, variance, min, max
+        allowed operators: mean, median, std_dev, sum, variance, min, max
     """
-    operators = ['mean', 'median', 'std_dev', 'variance', 'min', 'max']
+    operators = ['mean', 'median', 'std_dev', 'sum', 'variance', 'min', 'max']
     operator = operator.lower()
     if operator not in operators:
         raise ValueError("operator {} not recognised. "
@@ -66,4 +66,4 @@ def operator_accept_weights(operator):
         bool: True if operator support weights, False otherwise
 
     """
-    return operator.lower() in ('mean',)
+    return operator.lower() in ('mean', 'sum')
diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py
index 368815628..020d0d034 100644
--- a/esmvalcore/preprocessor/_time.py
+++ b/esmvalcore/preprocessor/_time.py
@@ -8,11 +8,11 @@ import logging
 from warnings import filterwarnings
 
 import cf_units
+import dask.array as da
 import iris
-import iris.util
 import iris.coord_categorisation
+import iris.util
 import numpy as np
-import dask.array as da
 
 from ._shared import get_iris_analysis_operation, operator_accept_weights
 
@@ -193,7 +193,7 @@ def daily_statistics(cube, operator='mean'):
 
     operator: str, optional
         Select operator to apply.
-        Available operators: 'mean', 'median', 'std_dev', 'min', 'max'
+        Available operators: 'mean', 'median', 'std_dev', 'sum', 'min', 'max'
 
     Returns
     -------
@@ -223,7 +223,7 @@ def monthly_statistics(cube, operator='mean'):
 
     operator: str, optional
         Select operator to apply.
-        Available operators: 'mean', 'median', 'std_dev', 'min', 'max'
+        Available operators: 'mean', 'median', 'std_dev', 'sum', 'min', 'max'
 
     Returns
     -------
@@ -253,7 +253,7 @@ def seasonal_statistics(cube, operator='mean'):
 
     operator: str, optional
         Select operator to apply.
-        Available operators: 'mean', 'median', 'std_dev', 'min', 'max'
+        Available operators: 'mean', 'median', 'std_dev', 'sum', 'min', 'max'
 
     Returns
     -------
@@ -263,13 +263,13 @@ def seasonal_statistics(cube, operator='mean'):
     if not cube.coords('clim_season'):
         iris.coord_categorisation.add_season(cube, 'time', name='clim_season')
     if not cube.coords('season_year'):
-        iris.coord_categorisation.add_season_year(
-            cube, 'time', name='season_year')
+        iris.coord_categorisation.add_season_year(cube,
+                                                  'time',
+                                                  name='season_year')
 
     operator = get_iris_analysis_operation(operator)
 
-    cube = cube.aggregated_by(['clim_season', 'season_year'],
-                              operator)
+    cube = cube.aggregated_by(['clim_season', 'season_year'], operator)
 
     # CMOR Units are days so we are safe to operate on days
     # Ranging on [90, 92] days makes this calendar-independent
@@ -308,7 +308,7 @@ def annual_statistics(cube, operator='mean'):
 
     operator: str, optional
         Select operator to apply.
-        Available operators: 'mean', 'median', 'std_dev', 'min', 'max'
+        Available operators: 'mean', 'median', 'std_dev', 'sum', 'min', 'max'
 
     Returns
     -------
@@ -340,7 +340,7 @@ def decadal_statistics(cube, operator='mean'):
 
     operator: str, optional
         Select operator to apply.
-        Available operators: 'mean', 'median', 'std_dev', min', 'max'
+        Available operators: 'mean', 'median', 'std_dev', 'sum', 'min', 'max'
 
     Returns
     -------
@@ -353,6 +353,7 @@ def decadal_statistics(cube, operator='mean'):
     operator = get_iris_analysis_operation(operator)
 
     if not cube.coords('decade'):
+
         def get_decade(coord, value):
             """Callback function to get decades from cube."""
             date = coord.units.num2date(value)
@@ -378,7 +379,7 @@ def climate_statistics(cube, operator='mean', period='full'):
 
     operator: str, optional
         Select operator to apply.
-        Available operators: 'mean', 'median', 'std_dev', 'min', 'max'
+        Available operators: 'mean', 'median', 'std_dev', 'sum', 'min', 'max'
 
     period: str, optional
         Period to compute the statistic over.
@@ -396,9 +397,9 @@ def climate_statistics(cube, operator='mean', period='full'):
         operator_method = get_iris_analysis_operation(operator)
         if operator_accept_weights(operator):
             time_weights = get_time_weights(cube)
-            cube = cube.collapsed(
-                'time', operator_method, weights=time_weights
-            )
+            cube = cube.collapsed('time',
+                                  operator_method,
+                                  weights=time_weights)
         else:
             cube = cube.collapsed('time', operator_method)
         return cube
@@ -450,9 +451,8 @@ def anomalies(cube, period):
     for i in range(cube_time.shape[0]):
         time = cube_time.points[i]
         indexes = cube_time.points == time
-        indexes = iris.util.broadcast_to_shape(
-            indexes, data.shape, (cube_coord_dim, )
-        )
+        indexes = iris.util.broadcast_to_shape(indexes, data.shape,
+                                               (cube_coord_dim, ))
         data[indexes] = data[indexes] - ref[cube_coord.points[i]]
 
     cube = cube.copy(data)
@@ -520,17 +520,13 @@ def regrid_time(cube, frequency):
         ]
     elif frequency == '3hr':
         cube.coord('time').cells = [
-            datetime.datetime(
-                t.year, t.month, t.day, t.hour - t.hour % 3, 0, 0
-            )
-            for t in time_c
+            datetime.datetime(t.year, t.month, t.day, t.hour - t.hour % 3, 0,
+                              0) for t in time_c
         ]
     elif frequency == '6hr':
         cube.coord('time').cells = [
-            datetime.datetime(
-                t.year, t.month, t.day, t.hour - t.hour % 6, 0, 0
-            )
-            for t in time_c
+            datetime.datetime(t.year, t.month, t.day, t.hour - t.hour % 6, 0,
+                              0) for t in time_c
         ]
 
     cube.coord('time').points = [


Test patch:
diff --git a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py
index 6daaacf8d..056a26d55 100644
--- a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py
+++ b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py
@@ -2,11 +2,10 @@
 import unittest
 
 from cf_units import Unit
-from iris.coords import DimCoord
-from iris.cube import Cube, CubeList
+from iris.cube import Cube
 
-from esmvalcore.cmor._fixes.cmip5.ec_earth import Sftlf, Sic, Tas
 from esmvalcore.cmor.fix import Fix
+from esmvalcore.cmor._fixes.cmip5.ec_earth import Sftlf, Sic
 
 
 class TestSic(unittest.TestCase):
@@ -47,64 +46,3 @@ class TestSftlf(unittest.TestCase):
         cube = self.fix.fix_data(self.cube)
         self.assertEqual(cube.data[0], 100)
         self.assertEqual(cube.units, Unit('J'))
-
-
-class TestTas(unittest.TestCase):
-    """Test tas fixes."""
-
-    def setUp(self):
-        """Prepare tests."""
-
-        height_coord = DimCoord(
-            2.,
-            standard_name='height',
-            long_name='height',
-            var_name='height',
-            units='m',
-            bounds=None,
-            attributes={'positive': 'up'}
-            )
-
-        time_coord = DimCoord(
-            1.,
-            standard_name='time',
-            var_name='time',
-            units=Unit('days since 2070-01-01 00:00:00', calendar='gregorian'),
-            )
-
-        self.height_coord = height_coord
-
-        self.cube_without = CubeList([Cube([3.0], var_name='tas')])
-        self.cube_without[0].add_aux_coord(time_coord, 0)
-
-        self.cube_with = CubeList([Cube([3.0], var_name='tas')])
-        self.cube_with[0].add_aux_coord(height_coord, ())
-        self.cube_with[0].add_aux_coord(time_coord, 0)
-        self.cube_with[0].coord('time').long_name = 'time'
-
-        self.fix = Tas()
-
-    def test_get(self):
-        """Test fix get"""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'EC-EARTH', 'tas'), [Tas()])
-
-    def test_tas_fix_metadata(self):
-        """Test metadata fix."""
-
-        out_cube_without = self.fix.fix_metadata(self.cube_without)
-
-        # make sure this does not raise an error
-        out_cube_with = self.fix.fix_metadata(self.cube_with)
-
-        coord = out_cube_without[0].coord('height')
-        assert coord == self.height_coord
-
-        coord = out_cube_without[0].coord('time')
-        assert coord.long_name == "time"
-
-        coord = out_cube_with[0].coord('height')
-        assert coord == self.height_coord
-
-        coord = out_cube_with[0].coord('time')
-        assert coord.long_name == "time"
diff --git a/tests/unit/preprocessor/_area/test_area.py b/tests/unit/preprocessor/_area/test_area.py
index 09a4b0345..ad029740a 100644
--- a/tests/unit/preprocessor/_area/test_area.py
+++ b/tests/unit/preprocessor/_area/test_area.py
@@ -86,6 +86,13 @@ class Test(tests.Test):
         expected = np.array([0.])
         self.assertArrayEqual(result.data, expected)
 
+    def test_area_statistics_sum(self):
+        """Test for sum of a 2D field."""
+        result = area_statistics(self.grid, 'sum')
+        grid_areas = iris.analysis.cartography.area_weights(self.grid)
+        expected = np.sum(grid_areas)
+        self.assertArrayEqual(result.data, expected)
+
     def test_area_statistics_variance(self):
         """Test for area average of a 2D field."""
         result = area_statistics(self.grid, 'variance')
diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py
index 660284eef..bbb2da64b 100644
--- a/tests/unit/preprocessor/_time/test_time.py
+++ b/tests/unit/preprocessor/_time/test_time.py
@@ -206,6 +206,42 @@ class TestClimatology(tests.Test):
         expected = np.array([1.])
         assert_array_equal(result.data, expected)
 
+    def test_time_sum(self):
+        """Test for time sum of a 1D field."""
+        data = np.ones((3))
+        data[1] = 2.0
+        times = np.array([15., 45., 75.])
+        bounds = np.array([[0., 30.], [30., 60.], [60., 90.]])
+        cube = self._create_cube(data, times, bounds)
+
+        result = climate_statistics(cube, operator='sum')
+        expected = np.array([120.])
+        assert_array_equal(result.data, expected)
+
+    def test_time_sum_uneven(self):
+        """Test for time sum of a 1D field with uneven time boundaries."""
+        data = np.array([1., 5.])
+        times = np.array([5., 25.])
+        bounds = np.array([[0., 1.], [1., 4.]])
+        cube = self._create_cube(data, times, bounds)
+
+        result = climate_statistics(cube, operator='sum')
+        expected = np.array([16.0])
+        assert_array_equal(result.data, expected)
+
+    def test_time_sum_365_day(self):
+        """Test for time sum of a realisitc time axis and 365 day calendar"""
+        data = np.ones((6, ))
+        data[3] = 2.0
+        times = np.array([15, 45, 74, 105, 135, 166])
+        bounds = np.array([[0, 31], [31, 59], [59, 90], [90, 120], [120, 151],
+                           [151, 181]])
+        cube = self._create_cube(data, times, bounds)
+
+        result = climate_statistics(cube, operator='sum')
+        expected = np.array([211.])
+        assert_array_equal(result.data, expected)
+
     def test_season_climatology(self):
         """Test for time avg of a realisitc time axis and 365 day calendar"""
         data = np.ones((6, ))
@@ -340,6 +376,16 @@ class TestSeasonalStatistics(tests.Test):
         expected = np.array([4., 7., 10.])
         assert_array_equal(result.data, expected)
 
+    def test_season_sum(self):
+        """Test for season sum of a 1D field."""
+        data = np.arange(12)
+        times = np.arange(15, 360, 30)
+        cube = self._create_cube(data, times)
+
+        result = seasonal_statistics(cube, 'sum')
+        expected = np.array([9., 18., 27.])
+        assert_array_equal(result.data, expected)
+
 
 class TestMonthlyStatistics(tests.Test):
     """Test :func:`esmvalcore.preprocessor._time.monthly_statistics`"""
@@ -362,7 +408,7 @@ class TestMonthlyStatistics(tests.Test):
 
         result = monthly_statistics(cube, 'mean')
         expected = np.array([
-            0.5,  2.5,  4.5,  6.5,  8.5, 10.5, 12.5, 14.5,
+            0.5, 2.5, 4.5, 6.5, 8.5, 10.5, 12.5, 14.5,
             16.5, 18.5, 20.5, 22.5
         ])
         assert_array_equal(result.data, expected)
@@ -400,6 +446,16 @@ class TestMonthlyStatistics(tests.Test):
         expected = np.array([1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23])
         assert_array_equal(result.data, expected)
 
+    def test_sum(self):
+        """Test sum of a 1D field."""
+        data = np.arange(24)
+        times = np.arange(7, 360, 15)
+        cube = self._create_cube(data, times)
+
+        result = monthly_statistics(cube, 'sum')
+        expected = np.array([1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45])
+        assert_array_equal(result.data, expected)
+
 
 class TestDailyStatistics(tests.Test):
     """Test :func:`esmvalcore.preprocessor._time.monthly_statistics`"""
@@ -454,6 +510,16 @@ class TestDailyStatistics(tests.Test):
         expected = np.array([3., 7.])
         assert_array_equal(result.data, expected)
 
+    def test_sum(self):
+        """Test sum of a 1D field."""
+        data = np.arange(8)
+        times = np.arange(0, 48, 6)
+        cube = self._create_cube(data, times)
+
+        result = daily_statistics(cube, 'sum')
+        expected = np.array([6., 22.])
+        assert_array_equal(result.data, expected)
+
 
 class TestRegridTimeMonthly(tests.Test):
     """Tests for regrid_time with monthly frequency."""
@@ -710,6 +776,20 @@ def test_annual_average(existing_coord):
     assert_array_equal(result.coord('time').points, expected_time)
 
 
+@pytest.mark.parametrize('existing_coord', [True, False])
+def test_annual_sum(existing_coord):
+    """Test for annual sum."""
+    cube = make_time_series(number_years=2)
+    if existing_coord:
+        iris.coord_categorisation.add_year(cube, 'time')
+
+    result = annual_statistics(cube, 'sum')
+    expected = np.array([12., 12.])
+    assert_array_equal(result.data, expected)
+    expected_time = np.array([180., 540.])
+    assert_array_equal(result.coord('time').points, expected_time)
+
+
 @pytest.mark.parametrize('existing_coord', [True, False])
 def test_decadal_average(existing_coord):
     """Test for decadal average."""
@@ -731,6 +811,27 @@ def test_decadal_average(existing_coord):
     assert_array_equal(result.coord('time').points, expected_time)
 
 
+@pytest.mark.parametrize('existing_coord', [True, False])
+def test_decadal_sum(existing_coord):
+    """Test for decadal average."""
+    cube = make_time_series(number_years=20)
+    if existing_coord:
+
+        def get_decade(coord, value):
+            """Callback function to get decades from cube."""
+            date = coord.units.num2date(value)
+            return date.year - date.year % 10
+
+        iris.coord_categorisation.add_categorised_coord(
+            cube, 'decade', 'time', get_decade)
+
+    result = decadal_statistics(cube, 'sum')
+    expected = np.array([120., 120.])
+    assert_array_equal(result.data, expected)
+    expected_time = np.array([1800., 5400.])
+    assert_array_equal(result.coord('time').points, expected_time)
+
+
 def make_map_data(number_years=2):
     """Make a cube with time, lat and lon dimensions."""
     times = np.arange(0.5, number_years * 360)