Repository: pydata/xarray

Issue:
DOC: Broken links from generated documentation to github
This is the link that is used when i click on the "Edit on Github" button in the documentation. 
https://github.com/pydata/xarray/blob/stable/doc/generated/xarray.DataArray.std.rst 

I wish to access the implementation of Dataset.std() because I am not able to reproduce the results when I calculate the sample standard deviation. Based on the formula below. 

STD = sqrt( sum( x - x.mean() )**2 / (n-1) ) 

Discussion:
dcherian: >  vendor github_url because it is only a small part of scanpydoc.

:+1:
max-sixty: Amazing! Thanks @keewis !

Shall I email RTD support re `DONT_OVERWRITE_SPHINX_CONTEXT` or do you want to?
keewis: > Shall I email RTD support re `DONT_OVERWRITE_SPHINX_CONTEXT` or do you want to?

sorry for not telling, I already did that directly after creating the PR
keewis: I looked at the source of `scanpydoc.rtd_github_links`. It seems to be a bit larger than I expected, and is also licensed as GPLv3, so I guess it should be fine to keep it as a docs-only dependency (is vendoring GPLv3-licensed code even possible for non-GPL projects?).
keewis: okay, so support enabled the flag, but RTD times out while downloading something in `plotting.rst`. Does anyone know what's going on?
dcherian: These natural earth URLs time out sometimes. 

It's timing out on the browser for me now: https://naciscdn.org/naturalearth/110m/physical/ne_110m_coastline.zip
keewis: yes, it seems https://naciscdn.org is down.
keewis: to fix this, should we try to use [cartopy_offlinedata](https://anaconda.org/conda-forge/cartopy_offlinedata)?
dcherian: Does that help on CI?
keewis: not sure, I'm [trying that](https://readthedocs.org/projects/xarray-keewis/builds/11963638/) right now

Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 16165288..5d091519 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,8 +1,8 @@
 # https://pre-commit.com/
 repos:
   # isort should run before black as black sometimes tweaks the isort output
-  - repo: https://github.com/timothycrosley/isort
-    rev: 5.4.2
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.5.4
     hooks:
       - id: isort
   # https://github.com/python/black#version-control-integration
@@ -11,7 +11,7 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/keewis/blackdoc
-    rev: v0.1.2
+    rev: v0.2
     hooks:
       - id: blackdoc
   - repo: https://gitlab.com/pycqa/flake8
diff --git a/ci/requirements/doc.yml b/ci/requirements/doc.yml
index 5206b815..4170569f 100644
--- a/ci/requirements/doc.yml
+++ b/ci/requirements/doc.yml
@@ -28,5 +28,6 @@ dependencies:
   - zarr>=2.4
   - pip
   - pip:
+      - scanpydoc
       # relative to this file. Needs to be editable to be accepted.
       - -e ../..
diff --git a/doc/_templates/autosummary/base.rst b/doc/_templates/autosummary/base.rst
new file mode 100644
index 00000000..e6726a71
--- /dev/null
+++ b/doc/_templates/autosummary/base.rst
@@ -0,0 +1,3 @@
+:github_url: {{ fullname | github_url | escape_underscores }}
+
+{% extends "!autosummary/base.rst" %}
\ No newline at end of file
diff --git a/doc/conf.py b/doc/conf.py
index e83768ff..3db0b356 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -14,11 +14,13 @@
 
 import datetime
 import os
+import pathlib
 import subprocess
 import sys
 from contextlib import suppress
 
 import sphinx_autosummary_accessors
+from jinja2.defaults import DEFAULT_FILTERS
 
 import xarray
 
@@ -78,6 +80,7 @@ extensions = [
     "IPython.sphinxext.ipython_console_highlighting",
     "nbsphinx",
     "sphinx_autosummary_accessors",
+    "scanpydoc.rtd_github_links",
 ]
 
 extlinks = {
@@ -97,6 +100,15 @@ You can run this notebook in a `live session <https://mybinder.org/v2/gh/pydata/
 """
 
 autosummary_generate = True
+
+# for scanpydoc's jinja filter
+project_dir = pathlib.Path(__file__).parent.parent
+html_context = {
+    "github_user": "pydata",
+    "github_repo": "xarray",
+    "github_version": "master",
+}
+
 autodoc_typehints = "none"
 
 napoleon_use_param = False
@@ -404,3 +416,11 @@ intersphinx_mapping = {
     "dask": ("https://docs.dask.org/en/latest", None),
     "cftime": ("https://unidata.github.io/cftime", None),
 }
+
+
+def escape_underscores(string):
+    return string.replace("_", r"\_")
+
+
+def setup(app):
+    DEFAULT_FILTERS["escape_underscores"] = escape_underscores
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
index 5ee67efb..6b944fd1 100644
--- a/doc/whats-new.rst
+++ b/doc/whats-new.rst
@@ -34,6 +34,15 @@ New Features
 Bug fixes
 ~~~~~~~~~
 
+- Fix bug where datetime64 times are silently changed to incorrect values if they are outside the valid date range for ns precision when provided in some other units (:issue:`4427`, :pull:`4454`).
+  By `Andrew Pauling <https://github.com/andrewpauling>`_
+- Fix silently overwriting the `engine` key when passing :py:func:`open_dataset` a file object
+  to an incompatible netCDF (:issue:`4457`). Now incompatible combinations of files and engines raise
+  an exception instead. By `Alessandro Amici <https://github.com/alexamici>`_.
+- The ``min_count`` argument to :py:meth:`DataArray.sum()` and :py:meth:`DataArray.prod()`
+  is now ignored when not applicable, i.e. when ``skipna=False`` or when ``skipna=None``
+  and the dtype does not have a missing value (:issue:`4352`).
+  By `Mathias Hauser <https://github.com/mathause>`_.
 
 Documentation
 ~~~~~~~~~~~~~
diff --git a/xarray/backends/api.py b/xarray/backends/api.py
index 9ea22295..434ea5a1 100644
--- a/xarray/backends/api.py
+++ b/xarray/backends/api.py
@@ -126,8 +126,7 @@ def _get_engine_from_magic_number(filename_or_obj):
         if filename_or_obj.tell() != 0:
             raise ValueError(
                 "file-like object read/write pointer not at zero "
-                "please close and reopen, or use a context "
-                "manager"
+                "please close and reopen, or use a context manager"
             )
         magic_number = filename_or_obj.read(8)
         filename_or_obj.seek(0)
@@ -136,17 +135,10 @@ def _get_engine_from_magic_number(filename_or_obj):
         engine = "scipy"
     elif magic_number.startswith(b"\211HDF\r\n\032\n"):
         engine = "h5netcdf"
-        if isinstance(filename_or_obj, bytes):
-            raise ValueError(
-                "can't open netCDF4/HDF5 as bytes "
-                "try passing a path or file-like object"
-            )
     else:
-        if isinstance(filename_or_obj, bytes) and len(filename_or_obj) > 80:
-            filename_or_obj = filename_or_obj[:80] + b"..."
         raise ValueError(
-            "{} is not a valid netCDF file "
-            "did you mean to pass a string for a path instead?".format(filename_or_obj)
+            f"{magic_number} is not the signature of any supported file format "
+            "did you mean to pass a string for a path instead?"
         )
     return engine
 
@@ -163,6 +155,14 @@ def _get_default_engine(path, allow_remote=False):
     return engine
 
 
+def _autodetect_engine(filename_or_obj):
+    if isinstance(filename_or_obj, str):
+        engine = _get_default_engine(filename_or_obj, allow_remote=True)
+    else:
+        engine = _get_engine_from_magic_number(filename_or_obj)
+    return engine
+
+
 def _get_backend_cls(engine):
     """Select open_dataset method based on current engine"""
     try:
@@ -175,10 +175,13 @@ def _get_backend_cls(engine):
 
 
 def _normalize_path(path):
-    if is_remote_uri(path):
-        return path
-    else:
-        return os.path.abspath(os.path.expanduser(path))
+    if isinstance(path, Path):
+        path = str(path)
+
+    if isinstance(path, str) and not is_remote_uri(path):
+        path = os.path.abspath(os.path.expanduser(path))
+
+    return path
 
 
 def _validate_dataset_names(dataset):
@@ -532,24 +535,13 @@ def open_dataset(
         ds2._file_obj = ds._file_obj
         return ds2
 
-    if isinstance(filename_or_obj, Path):
-        filename_or_obj = str(filename_or_obj)
+    filename_or_obj = _normalize_path(filename_or_obj)
 
     if isinstance(filename_or_obj, AbstractDataStore):
         store = filename_or_obj
     else:
-        if isinstance(filename_or_obj, str):
-            filename_or_obj = _normalize_path(filename_or_obj)
-
-            if engine is None:
-                engine = _get_default_engine(filename_or_obj, allow_remote=True)
-        elif engine != "zarr":
-            if engine not in [None, "scipy", "h5netcdf"]:
-                raise ValueError(
-                    "can only read bytes or file-like objects "
-                    "with engine='scipy' or 'h5netcdf'"
-                )
-            engine = _get_engine_from_magic_number(filename_or_obj)
+        if engine is None:
+            engine = _autodetect_engine(filename_or_obj)
 
         if engine in ["netcdf4", "h5netcdf"]:
             extra_kwargs["group"] = group
diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py
index f3e61eee..2d5292f5 100644
--- a/xarray/backends/h5netcdf_.py
+++ b/xarray/backends/h5netcdf_.py
@@ -121,6 +121,25 @@ class H5NetCDFStore(WritableCFDataStore):
     ):
         import h5netcdf
 
+        if isinstance(filename, bytes):
+            raise ValueError(
+                "can't open netCDF4/HDF5 as bytes "
+                "try passing a path or file-like object"
+            )
+        elif hasattr(filename, "tell"):
+            if filename.tell() != 0:
+                raise ValueError(
+                    "file-like object read/write pointer not at zero "
+                    "please close and reopen, or use a context manager"
+                )
+            else:
+                magic_number = filename.read(8)
+                filename.seek(0)
+                if not magic_number.startswith(b"\211HDF\r\n\032\n"):
+                    raise ValueError(
+                        f"{magic_number} is not the signature of a valid netCDF file"
+                    )
+
         if format not in [None, "NETCDF4"]:
             raise ValueError("invalid format for h5netcdf backend")
 
diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py
index 0a917cde..bd799d10 100644
--- a/xarray/backends/netCDF4_.py
+++ b/xarray/backends/netCDF4_.py
@@ -333,6 +333,12 @@ class NetCDF4DataStore(WritableCFDataStore):
     ):
         import netCDF4
 
+        if not isinstance(filename, str):
+            raise ValueError(
+                "can only read bytes or file-like objects "
+                "with engine='scipy' or 'h5netcdf'"
+            )
+
         if format is None:
             format = "NETCDF4"
 
diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py
index 53849a3e..f35dfd23 100644
--- a/xarray/core/duck_array_ops.py
+++ b/xarray/core/duck_array_ops.py
@@ -325,6 +325,8 @@ def _create_nan_agg_method(name, dask_module=dask_array, coerce_strings=False):
             nanname = "nan" + name
             func = getattr(nanops, nanname)
         else:
+            if name in ["sum", "prod"]:
+                kwargs.pop("min_count", None)
             func = _dask_or_eager_func(name, dask_module=dask_module)
 
         try:
@@ -361,7 +363,7 @@ median = _create_nan_agg_method("median", dask_module=dask_array_compat)
 median.numeric_only = True
 prod = _create_nan_agg_method("prod")
 prod.numeric_only = True
-sum.available_min_count = True
+prod.available_min_count = True
 cumprod_1d = _create_nan_agg_method("cumprod")
 cumprod_1d.numeric_only = True
 cumsum_1d = _create_nan_agg_method("cumsum")
diff --git a/xarray/core/parallel.py b/xarray/core/parallel.py
index ff2113f7..f3a75034 100644
--- a/xarray/core/parallel.py
+++ b/xarray/core/parallel.py
@@ -255,7 +255,9 @@ def map_blocks(
     to the function being applied in ``xr.map_blocks()``:
 
     >>> array.map_blocks(
-    ...     calculate_anomaly, kwargs={"groupby_type": "time.year"}, template=array,
+    ...     calculate_anomaly,
+    ...     kwargs={"groupby_type": "time.year"},
+    ...     template=array,
     ... )  # doctest: +ELLIPSIS
     <xarray.DataArray (time: 24)>
     dask.array<calculate_anomaly-...-<this, shape=(24,), dtype=float64, chunksize=(24,), chunktype=numpy.ndarray>
diff --git a/xarray/core/variable.py b/xarray/core/variable.py
index c55e61cb..f4ced459 100644
--- a/xarray/core/variable.py
+++ b/xarray/core/variable.py
@@ -177,7 +177,9 @@ def _maybe_wrap_data(data):
 
 def _possibly_convert_objects(values):
     """Convert arrays of datetime.datetime and datetime.timedelta objects into
-    datetime64 and timedelta64, according to the pandas convention.
+    datetime64 and timedelta64, according to the pandas convention. Also used for
+    validating that datetime64 and timedelta64 objects are within the valid date
+    range for ns precision, as pandas will raise an error if they are not.
     """
     return np.asarray(pd.Series(values.ravel())).reshape(values.shape)
 
@@ -238,16 +240,16 @@ def as_compatible_data(data, fastpath=False):
                     '"1"'
                 )
 
-    # validate whether the data is valid data types
+    # validate whether the data is valid data types.
     data = np.asarray(data)
 
     if isinstance(data, np.ndarray):
         if data.dtype.kind == "O":
             data = _possibly_convert_objects(data)
         elif data.dtype.kind == "M":
-            data = np.asarray(data, "datetime64[ns]")
+            data = _possibly_convert_objects(data)
         elif data.dtype.kind == "m":
-            data = np.asarray(data, "timedelta64[ns]")
+            data = _possibly_convert_objects(data)
 
     return _maybe_wrap_data(data)
 


Test patch:
diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index c9030e31..81664737 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -2224,7 +2224,7 @@ class TestGenericNetCDFData(CFEncodedBase, NetCDF3Only):
                 open_dataset(tmp_file, engine="foobar")
 
         netcdf_bytes = data.to_netcdf()
-        with raises_regex(ValueError, "can only read bytes or file-like"):
+        with raises_regex(ValueError, "unrecognized engine"):
             open_dataset(BytesIO(netcdf_bytes), engine="foobar")
 
     def test_cross_engine_read_write_netcdf3(self):
@@ -2494,13 +2494,13 @@ class TestH5NetCDFFileObject(TestH5NetCDFData):
         with raises_regex(ValueError, "HDF5 as bytes"):
             with open_dataset(b"\211HDF\r\n\032\n", engine="h5netcdf"):
                 pass
-        with raises_regex(ValueError, "not a valid netCDF"):
+        with raises_regex(ValueError, "not the signature of any supported file"):
             with open_dataset(b"garbage"):
                 pass
         with raises_regex(ValueError, "can only read bytes"):
             with open_dataset(b"garbage", engine="netcdf4"):
                 pass
-        with raises_regex(ValueError, "not a valid netCDF"):
+        with raises_regex(ValueError, "not the signature of a valid netCDF file"):
             with open_dataset(BytesIO(b"garbage"), engine="h5netcdf"):
                 pass
 
@@ -2526,11 +2526,23 @@ class TestH5NetCDFFileObject(TestH5NetCDFData):
                 with open_dataset(f, engine="h5netcdf") as actual:
                     assert_identical(expected, actual)
 
+                f.seek(0)
+                with open_dataset(f) as actual:
+                    assert_identical(expected, actual)
+
                 f.seek(0)
                 with BytesIO(f.read()) as bio:
                     with open_dataset(bio, engine="h5netcdf") as actual:
                         assert_identical(expected, actual)
 
+                f.seek(0)
+                with raises_regex(TypeError, "not a valid NetCDF 3"):
+                    open_dataset(f, engine="scipy")
+
+                f.seek(8)
+                with raises_regex(ValueError, "read/write pointer not at zero"):
+                    open_dataset(f)
+
 
 @requires_h5netcdf
 @requires_dask
diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py
index 41f03115..44b2ed23 100644
--- a/xarray/tests/test_duck_array_ops.py
+++ b/xarray/tests/test_duck_array_ops.py
@@ -580,15 +580,17 @@ def test_dask_gradient(axis, edge_order):
 @pytest.mark.parametrize("dask", [False, True])
 @pytest.mark.parametrize("func", ["sum", "prod"])
 @pytest.mark.parametrize("aggdim", [None, "x"])
-def test_min_count(dim_num, dtype, dask, func, aggdim):
+@pytest.mark.parametrize("contains_nan", [True, False])
+@pytest.mark.parametrize("skipna", [True, False, None])
+def test_min_count(dim_num, dtype, dask, func, aggdim, contains_nan, skipna):
     if dask and not has_dask:
         pytest.skip("requires dask")
 
-    da = construct_dataarray(dim_num, dtype, contains_nan=True, dask=dask)
+    da = construct_dataarray(dim_num, dtype, contains_nan=contains_nan, dask=dask)
     min_count = 3
 
-    actual = getattr(da, func)(dim=aggdim, skipna=True, min_count=min_count)
-    expected = series_reduce(da, func, skipna=True, dim=aggdim, min_count=min_count)
+    actual = getattr(da, func)(dim=aggdim, skipna=skipna, min_count=min_count)
+    expected = series_reduce(da, func, skipna=skipna, dim=aggdim, min_count=min_count)
     assert_allclose(actual, expected)
     assert_dask_array(actual, dask)
 
diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py
index efebe09e..08fe0739 100644
--- a/xarray/tests/test_variable.py
+++ b/xarray/tests/test_variable.py
@@ -294,6 +294,19 @@ class VariableSubclassobjects:
         actual = self.cls("x", data)
         assert actual.dtype == data.dtype
 
+    def test_datetime64_valid_range(self):
+        data = np.datetime64("1250-01-01", "us")
+        pderror = pd.errors.OutOfBoundsDatetime
+        with raises_regex(pderror, "Out of bounds nanosecond"):
+            self.cls(["t"], [data])
+
+    @pytest.mark.xfail(reason="pandas issue 36615")
+    def test_timedelta64_valid_range(self):
+        data = np.timedelta64("200000", "D")
+        pderror = pd.errors.OutOfBoundsTimedelta
+        with raises_regex(pderror, "Out of bounds nanosecond"):
+            self.cls(["t"], [data])
+
     def test_pandas_data(self):
         v = self.cls(["x"], pd.Series([0, 1, 2], index=[3, 2, 1]))
         assert_identical(v, v[[0, 1, 2]])