Repository: ESMValGroup/ESMValCore

Issue:
High value for 'branch_time_in_parent' attribute crashes CMOR check
The NESM3 model has a very high value for the `'branch_time_in_parent'` attribute (I'm not sure if this is correct):
```
$ ncdump /work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ/ScenarioMIP/NUIST/NESM3/ssp126/r1i1p1f1/Omon/tos/gn/v20190731/tos_Omon_NESM3_ssp126_r1i1p1f1_gn_201501-210012.nc -h
netcdf tos_Omon_NESM3_ssp126_r1i1p1f1_gn_201501-210012 {
dimensions:
	time = UNLIMITED ; // (1032 currently)
	j = 292 ;
	i = 362 ;
	bnds = 2 ;
	vertices = 4 ;
variables:
	double time(time) ;
		time:bounds = "time_bnds" ;
		time:units = "days since 2015-01-01 00:00:00" ;
		time:calendar = "365_day" ;
		time:axis = "T" ;
		time:long_name = "time" ;
		time:standard_name = "time" ;
	double time_bnds(time, bnds) ;
	int j(j) ;
		j:units = "1" ;
		j:long_name = "cell index along second dimension" ;
	int i(i) ;
		i:units = "1" ;
		i:long_name = "cell index along first dimension" ;
	float latitude(j, i) ;
		latitude:standard_name = "latitude" ;
		latitude:long_name = "latitude" ;
		latitude:units = "degrees_north" ;
		latitude:missing_value = 1.e+20f ;
		latitude:_FillValue = 1.e+20f ;
		latitude:bounds = "vertices_latitude" ;
	float longitude(j, i) ;
		longitude:standard_name = "longitude" ;
		longitude:long_name = "longitude" ;
		longitude:units = "degrees_east" ;
		longitude:missing_value = 1.e+20f ;
		longitude:_FillValue = 1.e+20f ;
		longitude:bounds = "vertices_longitude" ;
	float vertices_latitude(j, i, vertices) ;
		vertices_latitude:units = "degrees_north" ;
		vertices_latitude:missing_value = 1.e+20f ;
		vertices_latitude:_FillValue = 1.e+20f ;
	float vertices_longitude(j, i, vertices) ;
		vertices_longitude:units = "degrees_east" ;
		vertices_longitude:missing_value = 1.e+20f ;
		vertices_longitude:_FillValue = 1.e+20f ;
	float tos(time, j, i) ;
		tos:standard_name = "sea_surface_temperature" ;
		tos:long_name = "Sea Surface Temperature" ;
		tos:comment = "Temperature of upper boundary of the liquid ocean, including temperatures below sea-ice and floating ice shelves." ;
		tos:units = "degC" ;
		tos:cell_methods = "area: mean where sea time: mean (interval: 1 month)" ;
		tos:cell_measures = "area: areacello" ;
		tos:missing_value = 1.e+20f ;
		tos:_FillValue = 1.e+20f ;
		tos:coordinates = "latitude longitude" ;

// global attributes:
		:Conventions = "CF-1.7 CMIP-6.2" ;
		:activity_id = "ScenarioMIP" ;
		:branch_method = "Standard" ;
		:branch_time_in_child = 0. ;
		:branch_time_in_parent = 5238432000. ;
		:contact = "esmc@nuist.edu.cn" ;
		:creation_date = "2019-07-30T23:50:30Z" ;
		:data_specs_version = "01.00.30" ;
		:experiment = "update of RCP2.6 based on SSP1" ;
		:experiment_id = "ssp126" ;
		:external_variables = "areacello" ;
		:forcing_index = 1 ;
		:frequency = "mon" ;
		:further_info_url = "https://furtherinfo.es-doc.org/CMIP6.NUIST.NESM3.ssp126.none.r1i1p1f1" ;
		:grid = "ORCA1 tripolar grid, 1 deg with refinement to 1/3 deg within 20 degrees of the equator; 362 x 292 longitude/latitude; 46 vertical levels" ;
		:grid_label = "gn" ;
		:history = "2019-07-30T23:50:30Z ; CMOR rewrote data to be consistent with CMIP6, CF-1.7 CMIP-6.2 and CF standards." ;
		:initialization_index = 1 ;
		:institution = "Nanjing University of Information Science and Technology, Nanjing, 210044, China" ;
		:institution_id = "NUIST" ;
		:mip_era = "CMIP6" ;
		:nominal_resolution = "100 km" ;
		:parent_activity_id = "CMIP" ;
		:parent_experiment_id = "historical" ;
		:parent_mip_era = "CMIP6" ;
		:parent_source_id = "NESM3" ;
		:parent_time_units = "days since 2015-01-01 00:00:0.0" ;
		:parent_variant_label = "r1i1p1f1" ;
		:physics_index = 1 ;
		:product = "model-output" ;
		:realization_index = 1 ;
		:realm = "ocean" ;
		:references = "Model described by Cao et al. 2018 GMD   Also see https://www.geosci-model-dev.net/11/2975/2018/gmd-11-2975-2018.pdf" ;
		:run_variant = "historical run" ;
		:source = "NESM v3 (2016): \n",
			"aerosol: none\n",
			"atmos: ECHAM v6.3.02 (T63; 192 x 96 longitude/latitude; 47 levels; top level 1 Pa)\n",
			"atmosChem: none\n",
			"land: JSBACH v3.1\n",
			"landIce: none\n",
			"ocean: NEMO v3.4 (NEMO v3.4, tripolar primarily 1deg; 362 x 292 longitude/latitude; 46 levels; top grid cell 0-6 m)\n",
			"ocnBgchem: none\n",
			"seaIce: CICE4.1 (384 x 362 longitude/latitude)" ;
		:source_id = "NESM3" ;
		:source_type = "AOGCM" ;
		:sub_experiment = "none" ;
		:sub_experiment_id = "none" ;
		:table_id = "Omon" ;
		:table_info = "Creation Date:(30 April 2019) MD5:43ed50479c79f31d44ce77afda8daad6" ;
		:title = "NESM3 output prepared for CMIP6" ;
		:tracking_id = "hdl:21.14100/86496cd8-fd39-4eb8-981a-09eff139e3df" ;
		:variable_id = "tos" ;
		:variant_label = "r1i1p1f1" ;
		:license = "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law." ;
		:cmor_version = "3.4.0" ;
}
```
and this leads to a crash in the CMOR checker:
```python
In [1]: from esmvalcore.dataset import Dataset

In [2]: ds = Dataset(
    ...:   dataset='NESM3',
    ...:   exp='ssp126',
    ...:   ensemble='r1i1p1f1',
    ...:   mip='Omon',
    ...:   short_name='tos',
    ...:   grid='gn',
    ...:   project='CMIP6',
    ...: )

In [3]: ds.load()
ERROR:esmvalcore.preprocessor:Failed to run preprocessor function 'fix_metadata' on the data
[<iris 'Cube' of sea_surface_temperature / (degC) (time: 1032; cell index along second dimension: 292; cell index along first dimension: 362)>]
loaded from original input file(s)
[LocalFile('/home/bandela/climate_data/CMIP6/ScenarioMIP/NUIST/NESM3/ssp126/r1i1p1f1/Omon/tos/gn/v20190731/tos_Omon_NESM3_ssp126_r1i1p1f1_gn_201501-210012.nc')]
with function argument(s)
check_level = <CheckLevels.DEFAULT: 3>,
session = Session({'auxiliary_data_dir': PosixPath('/home/bandela/auxiliary_data'),
         'check_level': <CheckLevels.DEFAULT: 3>,
         'compress_netcdf': False,
         'config_developer_file': PosixPath('/home/bandela/src/esmvalgroup/esmvalcore/esmvalcore/config-developer.yml'),
         'config_file': PosixPath('/home/bandela/.esmvaltool/config-user.yml'),
         'diagnostics': None,
         'download_dir': PosixPath('/home/bandela/climate_data'),
         'drs': {'CMIP3': 'ESGF',
                 'CMIP5': 'ESGF',
                 'CMIP6': 'ESGF',
                 'CORDEX': 'ESGF',
                 'obs4MIPs': 'ESGF'},
         'exit_on_warning': False,
         'extra_facets_dir': (),
         'log_level': 'info',
         'max_datasets': None,
         'max_parallel_tasks': None,
         'max_years': None,
         'output_dir': PosixPath('/home/bandela/esmvaltool_output'),
         'output_file_type': 'png',
         'profile_diagnostic': False,
         'remove_preproc_dir': True,
         'resume_from': [],
         'rootpath': {'default': [PosixPath('/home/bandela/climate_data')]},
         'run_diagnostic': True,
         'save_intermediary_cubes': False,
         'search_esgf': 'always',
         'skip_nonexistent': False}),
dataset = 'NESM3',
exp = 'ssp126',
ensemble = 'r1i1p1f1',
mip = 'Omon',
short_name = 'tos',
grid = 'gn',
project = 'CMIP6',
institute = ['NUIST'],
activity = 'ScenarioMIP',
original_short_name = 'tos',
standard_name = 'sea_surface_temperature',
long_name = 'Sea Surface Temperature',
units = 'degC',
modeling_realm = ['ocean'],
frequency = 'mon'
---------------------------------------------------------------------------
OverflowError                             Traceback (most recent call last)
Cell In[35], line 1
----> 1 ds.load()

File ~/src/esmvalgroup/esmvalcore/esmvalcore/dataset.py:680, in Dataset.load(self)
    667 def load(self) -> Cube:
    668     """Load dataset.
    669 
    670     Raises
   (...)
    678         An :mod:`iris` cube with the data corresponding the the dataset.
    679     """
--> 680     return self._load_with_callback(callback='default')

File ~/src/esmvalgroup/esmvalcore/esmvalcore/dataset.py:689, in Dataset._load_with_callback(self, callback)
    686     input_files.extend(supplementary_dataset.files)
    687 esgf.download(input_files, self.session['download_dir'])
--> 689 cube = self._load(callback)
    690 supplementary_cubes = []
    691 for supplementary_dataset in self.supplementaries:

File ~/src/esmvalgroup/esmvalcore/esmvalcore/dataset.py:774, in Dataset._load(self, callback)
    769 result = [
    770     file.local_file(self.session['download_dir']) if isinstance(
    771         file, esgf.ESGFFile) else file for file in self.files
    772 ]
    773 for step, kwargs in settings.items():
--> 774     result = preprocess(
    775         result,
    776         step,
    777         input_files=self.files,
    778         output_file=output_file,
    779         debug=self.session['save_intermediary_cubes'],
    780         **kwargs,
    781     )
    783 cube = result[0]
    784 return cube

File ~/src/esmvalgroup/esmvalcore/esmvalcore/preprocessor/__init__.py:375, in preprocess(items, step, input_files, output_file, debug, **settings)
    373 result = []
    374 if itype.endswith('s'):
--> 375     result.append(_run_preproc_function(function, items, settings,
    376                                         input_files=input_files))
    377 else:
    378     for item in items:

File ~/src/esmvalgroup/esmvalcore/esmvalcore/preprocessor/__init__.py:328, in _run_preproc_function(function, items, kwargs, input_files)
    323 logger.debug(
    324     "Running preprocessor function '%s' on the data\n%s%s\nwith function "
    325     "argument(s)\n%s", function.__name__, pformat(items), file_msg,
    326     kwargs_str)
    327 try:
--> 328     return function(items, **kwargs)
    329 except Exception:
    330     # To avoid very long error messages, we truncate the arguments and
    331     # input files here at a given threshold
    332     n_shown_args = 4

File ~/src/esmvalgroup/esmvalcore/esmvalcore/cmor/fix.py:173, in fix_metadata(cubes, short_name, project, dataset, mip, frequency, check_level, session, **extra_facets)
    165 cube = _get_single_cube(cube_list, short_name, project, dataset)
    166 checker = _get_cmor_checker(frequency=frequency,
    167                             table=project,
    168                             mip=mip,
   (...)
    171                             fail_on_error=False,
    172                             automatic_fixes=True)
--> 173 cube = checker(cube).check_metadata()
    174 cube.attributes.pop('source_file', None)
    175 fixed_cubes.append(cube)

File ~/src/esmvalgroup/esmvalcore/esmvalcore/cmor/check.py:189, in CMORCheck.check_metadata(self, logger)
    187 self._check_coords()
    188 if self.frequency != 'fx':
--> 189     self._check_time_coord()
    191 self._check_rank()
    193 self.report_debug_messages()

File ~/src/esmvalgroup/esmvalcore/esmvalcore/cmor/check.py:882, in CMORCheck._check_time_coord(self)
    880 branch_parent = 'branch_time_in_parent'
    881 if branch_parent in attrs:
--> 882     attrs[branch_parent] = parent_units.convert(
    883         attrs[branch_parent], coord.units)
    884 branch_child = 'branch_time_in_child'
    885 if branch_child in attrs:

File ~/mambaforge/envs/esmvaltool/lib/python3.10/site-packages/cf_units/__init__.py:1842, in Unit.convert(self, value, other, ctype, inplace)
   1839 # Use cftime for converting reference times that are not using a
   1840 # gregorian calendar as it handles these and udunits does not.
   1841 if self.is_time_reference() and self.calendar != CALENDAR_STANDARD:
-> 1842     result_datetimes = cftime.num2date(
   1843         result, self.cftime_unit, self.calendar
   1844     )
   1845     result = cftime.date2num(
   1846         result_datetimes, other.cftime_unit, other.calendar
   1847     )
   1848     convert_type = isinstance(
   1849         value, np.ndarray
   1850     ) and np.issubsctype(value.dtype, np.floating)

File src/cftime/_cftime.pyx:617, in cftime._cftime.num2date()

File src/cftime/_cftime.pyx:414, in cftime._cftime.cast_to_int()

OverflowError: time values outside range of 64 bit signed integers
```


Discussion:
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/ESMValGroup/ESMValCore/pull/2668?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ESMValGroup) Report
Attention: Patch coverage is `50.00000%` with `4 lines` in your changes missing coverage. Please review.
> Project coverage is 95.05%. Comparing base [(`434e65e`)](https://app.codecov.io/gh/ESMValGroup/ESMValCore/commit/434e65ea895e758ccd998aa7d389d90dfb953d8d?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ESMValGroup) to head [(`9a23990`)](https://app.codecov.io/gh/ESMValGroup/ESMValCore/commit/9a239904ce29c1f113a7fd7bf7c5d52a4eceeb32?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ESMValGroup).

| [Files with missing lines](https://app.codecov.io/gh/ESMValGroup/ESMValCore/pull/2668?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ESMValGroup) | Patch % | Lines |
|---|---|---|
| [esmvalcore/cmor/\_fixes/fix.py](https://app.codecov.io/gh/ESMValGroup/ESMValCore/pull/2668?src=pr&el=tree&filepath=esmvalcore%2Fcmor%2F_fixes%2Ffix.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ESMValGroup#diff-ZXNtdmFsY29yZS9jbW9yL19maXhlcy9maXgucHk=) | 50.00% | [4 Missing :warning: ](https://app.codecov.io/gh/ESMValGroup/ESMValCore/pull/2668?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ESMValGroup) |

:x: Your patch check has failed because the patch coverage (50.00%) is below the target coverage (100.00%). You can increase the patch coverage or adjust the [target](https://docs.codecov.com/docs/commit-status#target) coverage.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #2668      +/-   ##
==========================================
- Coverage   95.07%   95.05%   -0.02%     
==========================================
  Files         255      255              
  Lines       14887    14893       +6     
==========================================
+ Hits        14154    14157       +3     
- Misses        733      736       +3     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/ESMValGroup/ESMValCore/pull/2668?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ESMValGroup).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ESMValGroup).

bouweandela: It looks like the readthedocs build is failing because of a newer version of sphinx 8.2.0. The last successful build was with 8.1.3.
valeriupredoi: Ah sounds about right - about the right time - beats me, especially after eight pints at the pub 😁🍺
valeriupredoi: those were 7 pints - and I forgot I'd approved it, thanks bud :beer: 

Patch:
diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py
index 9a229b2dc..4453177f7 100644
--- a/esmvalcore/cmor/_fixes/fix.py
+++ b/esmvalcore/cmor/_fixes/fix.py
@@ -831,15 +831,23 @@ class GenericFix(Fix):
 
                     branch_parent = "branch_time_in_parent"
                     if branch_parent in attrs:
-                        attrs[branch_parent] = parent_units.convert(
-                            attrs[branch_parent], cube_coord.units
-                        )
+                        try:
+                            attrs[branch_parent] = parent_units.convert(
+                                attrs[branch_parent], cube_coord.units
+                            )
+                        except OverflowError:
+                            # This happens when the time is very large.
+                            pass
 
                     branch_child = "branch_time_in_child"
                     if branch_child in attrs:
-                        attrs[branch_child] = old_units.convert(
-                            attrs[branch_child], cube_coord.units
-                        )
+                        try:
+                            attrs[branch_child] = old_units.convert(
+                                attrs[branch_child], cube_coord.units
+                            )
+                        except OverflowError:
+                            # This happens when the time is very large.
+                            pass
 
     def _fix_time_bounds(self, cube: Cube, cube_coord: Coord) -> None:
         """Fix time bounds."""


Test patch:
diff --git a/tests/integration/cmor/test_fix.py b/tests/integration/cmor/test_fix.py
index 4ce1b01e6..16dc1d492 100644
--- a/tests/integration/cmor/test_fix.py
+++ b/tests/integration/cmor/test_fix.py
@@ -734,18 +734,42 @@ class TestGenericFix:
         assert self.mock_debug.call_count == 3
         assert self.mock_warning.call_count == 7
 
-    def test_fix_metadata_amon_tas_invalid_time_attrs(self):
+    @pytest.mark.parametrize("problem", ["none", "bad_units", "bad_values"])
+    def test_fix_metadata_amon_tas_invalid_time_attrs(self, problem):
         """Test ``fix_metadata`` with invalid time attributes."""
         short_name = "tas"
         project = "CMIP6"
         dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__"
         mip = "Amon"
 
-        self.cubes_2d_latlon[0].attributes = {
-            "parent_time_units": "this is certainly not a unit",
-            "branch_time_in_parent": "BRANCH TIME IN PARENT",
-            "branch_time_in_child": "BRANCH TIME IN CHILD",
-        }
+        attributes = {}
+        expected_attributes = {}
+        if problem == "bad_units":
+            attributes["parent_time_units"] = "this is certainly not a unit"
+        else:
+            attributes["parent_time_units"] = "days since 1000-01-01 00:00:0.0"
+
+        if problem == "bad_values":
+            attributes["branch_time_in_child"] = 1e9
+            attributes["branch_time_in_parent"] = 1e9
+        else:
+            attributes["branch_time_in_child"] = 0.0
+            attributes["branch_time_in_parent"] = 91676.0
+
+        if problem == "none":
+            expected_attributes = {
+                "parent_time_units": "days since 1850-1-1 00:00:00",
+                "branch_time_in_child": 365,
+                "branch_time_in_parent": -218574,
+            }
+        else:
+            expected_attributes = attributes.copy()
+        if problem == "bad_values":
+            expected_attributes["parent_time_units"] = (
+                "days since 1850-1-1 00:00:00"
+            )
+
+        self.cubes_2d_latlon[0].attributes = attributes
 
         fixed_cubes = fix_metadata(
             self.cubes_2d_latlon,
@@ -762,11 +786,7 @@ class TestGenericFix:
         self.assert_lat_metadata(fixed_cube)
         self.assert_lon_metadata(fixed_cube)
 
-        assert fixed_cube.attributes == {
-            "parent_time_units": "this is certainly not a unit",
-            "branch_time_in_parent": "BRANCH TIME IN PARENT",
-            "branch_time_in_child": "BRANCH TIME IN CHILD",
-        }
+        assert fixed_cube.attributes == expected_attributes
 
         cmor_check_metadata(fixed_cube, project, mip, short_name)