Repository: ecmwf/earthkit-data
Pull request #148 (2023-07-17 12:21:58)

134 add inputs transformer tools for earthkit subpackages
Inputs transformer for use in sub-packages.
It uses type-setting, or explicit mapping, to try to ensure that the inputs passed to a function are converted to the appropriate type for that method. This means that earthkit users do not have to worry about the format of their data.

It is also possible for the user to bypass the conversion at the point of execution by setting the kwarg `_auto_convert_inputs = False`.

In the first instance this will be used earthkit-climate, but with further developments could/should be used by all earthkit subpackages


Patch:
diff --git a/docs/examples.rst b/docs/examples.rst
index 818a4c7..ae38526 100644
--- a/docs/examples.rst
+++ b/docs/examples.rst
@@ -11,6 +11,7 @@ Here is a list of example notebooks to illustrate how to use earthkit-data.
     :caption: Retrieve
 
     examples/mars.ipynb
+    examples/ads.ipynb
     examples/cds.ipynb
     examples/ecmwf_open_data.ipynb
     examples/fdb.ipynb
diff --git a/docs/examples/ads.ipynb b/docs/examples/ads.ipynb
new file mode 100644
index 0000000..3196ed9
--- /dev/null
+++ b/docs/examples/ads.ipynb
@@ -0,0 +1,219 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7ec282fb-fdf9-45ba-b3f7-3d46ef212e98",
+   "metadata": {},
+   "source": [
+    "## Retrieving CAMS data from the ADS"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "52ed2e76-9c54-4a85-b932-6604efc7ae85",
+   "metadata": {},
+   "source": [
+    "The \"ads\" data source provides access to the [Copernicus Atmosphere Data Store (ADS)](https://ads.atmosphere.copernicus.eu)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "f24509e0-dd8c-4032-9a3c-5a963ad544d9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import earthkit.data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9f4ce2c9-aaa3-4b76-aacb-1f597df3e338",
+   "metadata": {},
+   "source": [
+    "### Getting GRIB data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "be1b53fd-4871-4ebf-ae6c-a62fd1117082",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = earthkit.data.from_source(\n",
+    "    \"ads\",\n",
+    "    \"cams-global-reanalysis-eac4\",\n",
+    "    variable=[\"particulate_matter_10um\", \"particulate_matter_1um\"],\n",
+    "    area=[50, -50, 20, 50], #N,W,S,E \n",
+    "    date=\"2012-05-10\",\n",
+    "    time=\"12:00\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "09210614-fad6-46cf-915d-aae4a592d2d5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>pm10</td>\n",
+       "      <td>surface</td>\n",
+       "      <td>0</td>\n",
+       "      <td>20120510</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>pm1</td>\n",
+       "      <td>surface</td>\n",
+       "      <td>0</td>\n",
+       "      <td>20120510</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName typeOfLevel  level  dataDate  dataTime stepRange dataType  \\\n",
+       "0   ecmf      pm10     surface      0  20120510      1200         0       an   \n",
+       "1   ecmf       pm1     surface      0  20120510      1200         0       an   \n",
+       "\n",
+       "   number    gridType  \n",
+       "0       0  regular_ll  \n",
+       "1       0  regular_ll  "
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds.ls()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "eed0bfe1-eb81-4c26-a48b-5150f67bbb79",
+   "metadata": {},
+   "source": [
+    "### Getting NetCDF data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "47f368bd-64c7-4268-bacf-7f3f1aad1b24",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds1 = earthkit.data.from_source(\n",
+    "    \"ads\",\n",
+    "    \"cams-global-reanalysis-eac4\",\n",
+    "    variable=[\"particulate_matter_10um\", \"particulate_matter_1um\"],\n",
+    "    area=[50, -50, 20, 50], #N,W,S,E \n",
+    "    date=\"2012-05-10\",\n",
+    "    time=\"12:00\",\n",
+    "    format=\"netcdf\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "71d9a8f0-5cc7-45ce-a748-44791d9d1220",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "NetCDFField['pm10',[[time:0=1336651200000000000]]]\n",
+      "NetCDFField['pm1',[[time:0=1336651200000000000]]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for f in ds1:\n",
+    "    print(f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5514f18a-0441-4995-8451-4f4fdd54d60f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "dev",
+   "language": "python",
+   "name": "dev"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/docs/guide/sources.rst b/docs/guide/sources.rst
index c5f2947..e823475 100644
--- a/docs/guide/sources.rst
+++ b/docs/guide/sources.rst
@@ -40,6 +40,8 @@ We can get data from a given source by using :func:`from_source`:
       - retrieve data from the ECMWF `MARS archive <https://confluence.ecmwf.int/display/UDOC/MARS+user+documentation>`_
     * - :ref:`data-sources-cds`
       - retrieve data from the `Copernicus Climate Data Store <https://cds.climate.copernicus.eu/>`_ (CDS)
+    * - :ref:`data-sources-ads`
+      - retrieve data from the `Copernicus Atmosphere Data Store <https://ads.atmosphere.copernicus.eu/>`_ (ADS)
     * - :ref:`data-sources-eod`
       - retrieve `ECMWF open data <https://www.ecmwf.int/en/forecasts/datasets/open-data>`_
     * - :ref:`data-sources-fdb`
@@ -359,13 +361,51 @@ cds
 
   Data downloaded from the CDS is stored in the the :ref:`cache <caching>`.
 
-  To access data from the CDS, you will need to register and retrieve an access token. The process is described `here <https://cds.climate.copernicus.eu/api-how-to>`__. For more information, see the CDS `knowledge base`_.
+  To access data from the CDS, you will need to register and retrieve an access token. The process is described `here <https://cds.climate.copernicus.eu/api-how-to>`__. For more information, see the `CDS_knowledge base`_.
 
   Further examples:
 
       - :ref:`/examples/cds.ipynb`
 
 
+.. _data-sources-ads:
+
+ads
+---
+
+.. py:function:: from_source("ads", dataset, *args, **kwargs)
+  :noindex:
+
+  The ``ads`` source accesses the `Copernicus Atmosphere Data Store`_ (ADS), using the cdsapi_ package. In addition to data retrieval, ``request`` also has post-processing options such as ``grid`` and ``area`` for regridding and sub-area extraction respectively.
+
+  :param str dataset: the name of the ADS dataset
+  :param tuple *args: specifies the request as a dict
+  :param dict **kwargs: other keyword arguments specifying the request
+
+  The following example retrieves CAMS global reanalysis GRIB data for 2 parameters:
+
+  .. code-block:: python
+
+      import earthkit.data
+
+      ds = earthkit.data.from_source(
+          "ads",
+          "cams-global-reanalysis-eac4",
+          variable=["particulate_matter_10um", "particulate_matter_1um"],
+          area=[50, -50, 20, 50],  # N,W,S,E
+          date="2012-12-12",
+          time="12:00",
+      )
+
+  Data downloaded from the ADS is stored in the the :ref:`cache <caching>`.
+
+  To access data from the ADS, you will need to register and retrieve an access token. The process is described `here <https://ads.atmosphere.copernicus.eu/api-how-to>`__. For more information, see the `ADS_knowledge base`_.
+
+  Further examples:
+
+      - :ref:`/examples/ads.ipynb`
+
+
 .. _data-sources-eod:
 
 ecmwf-open-data
@@ -490,7 +530,9 @@ fdb
 .. _web API: https://www.ecmwf.int/en/forecasts/access-forecasts/ecmwf-web-api
 
 .. _Copernicus Climate Data Store: https://cds.climate.copernicus.eu/
+.. _Copernicus Atmosphere Data Store: https://ads.atmosphere.copernicus.eu/
 .. _cdsapi: https://pypi.org/project/cdsapi/
-.. _knowledge base: https://confluence.ecmwf.int/display/CKB/Copernicus+Knowledge+Base
+.. _CDS_knowledge base: https://confluence.ecmwf.int/pages/viewpage.action?pageId=151530614
+.. _ADS_knowledge base: https://confluence.ecmwf.int/pages/viewpage.action?pageId=151530675
 
 .. _ECMWF open data: https://www.ecmwf.int/en/forecasts/datasets/open-data
diff --git a/earthkit/data/input_manager.py b/earthkit/data/input_manager.py
deleted file mode 100644
index dd12df5..0000000
--- a/earthkit/data/input_manager.py
+++ /dev/null
@@ -1,179 +0,0 @@
-# (C) Copyright 2021 ECMWF.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-#
-import logging
-
-from .argument import Argument
-from .transformers import AvailabilityChecker, KwargsAliasTransformer
-
-LOG = logging.getLogger(__name__)
-
-
-class InputManager:
-    def __init__(
-        self,
-        decorators,
-    ):
-        self.decorators = decorators
-        self._pipeline = None
-        self.alias_argument = None
-        self.alias_arguments = []
-        self.availabilities = []
-        self.arguments = {}
-
-    @property
-    def pipeline(self):
-        if self._pipeline is None:
-            self.build_pipeline()
-        return self._pipeline
-
-    def register_availability(self, decorator):
-        for name, values in decorator.availability.unique_values().items():
-            if name not in self.arguments:
-                self.arguments[name] = Argument(name)
-            self.arguments[name].availability = values
-        self.availabilities.append(decorator.availability)
-
-    def register_normalize(self, decorator):
-        if decorator.name not in self.arguments:
-            self.arguments[decorator.name] = Argument(decorator.name)
-        self.arguments[decorator.name].normalize = decorator.kwargs
-
-    def register_alias_argument(self, decorator):
-        self.alias_arguments.append(decorator)
-
-    def build_pipeline(self):
-        self._pipeline = []
-        LOG.debug("Building arguments from decorators:\n %s", self.decorators)
-
-        self.arguments = {}
-
-        for decorator in self.decorators:
-            decorator.register(self)
-
-        self.arguments = list(self.arguments.values())
-
-        for a in self.alias_arguments:
-            transform = KwargsAliasTransformer(a)
-            self._pipeline.append(transform)
-
-        for a in self.arguments:
-            a.add_alias_transformers(self._pipeline)
-
-        for a in self.arguments:
-            a.add_type_transformers(self._pipeline)
-
-        for a in self.arguments:
-            a.add_alias_transformers(self._pipeline)
-
-        for availability in self.availabilities:
-            transform = AvailabilityChecker(availability)
-            self._pipeline.append(transform)
-
-        for a in self.arguments:
-            a.add_format_transformers(self._pipeline)
-
-    def apply_to_kwargs_before_default(self, kwargs):
-        LOG.debug(
-            f"Apply pipeline to kwargs before resolving default values: {safe_to_str(kwargs)}"
-        )
-        for t in self.pipeline:
-            if hasattr(t, "name"):
-                LOG.debug(f" - {t.name}: apply {t}.")
-            else:
-                LOG.debug(f" - apply {t}.")
-
-            kwargs = t.execute_before_default(kwargs)
-            LOG.debug(f"       kwargs: {safe_to_str(kwargs)}")
-
-        return kwargs
-
-    def apply_to_kwargs(self, kwargs):
-        LOG.debug(f"Apply pipeline to kwargs: {safe_to_str(kwargs)}")
-        for t in self.pipeline:
-            if hasattr(t, "name"):
-                LOG.debug(f" - {t.name}: apply {t}.")
-            else:
-                LOG.debug(f" - apply {t}.")
-
-            kwargs = t.execute(kwargs)
-            LOG.debug(f"       kwargs: {safe_to_str(kwargs)}")
-
-        return kwargs
-
-    def apply_to_value(self, value):
-        for t in self.pipeline:
-            value = t.__call__(value)
-        return value
-
-    def __repr__(self) -> str:
-        txt = "ARGUMENTS:[\n"
-        txt += " availability >>>\n"
-        if self.availabilities:
-            txt += f"{self.availabilities}<<<"
-        else:
-            txt += "None<<<"
-        txt += "\n"
-        for a in self.arguments:
-            txt += f"  {a}\n"
-        txt += "]"
-        if self._pipeline is None:
-            txt += "Pipeline[not-ready]"
-        else:
-            txt += "Pipeline:[\n"
-            for t in self._pipeline:
-                txt += f"  {t}\n"
-            txt += "]"
-        return txt
-
-    def get_argument(self, name):
-        for a in self.arguments:
-            if a == name:
-                return a
-        return None
-
-    def consolidate_defaults(self, defaults):
-        for k, default in defaults.items():
-            a = self.get_argument(k)
-            if a:
-                a.set_default(default)
-
-    def apply_to_arg_kwargs(self, args, kwargs, func):
-        from earthkit.data.arguments.args_kwargs import ArgsKwargs
-
-        args_kwargs = ArgsKwargs(args, kwargs, func=func)
-        LOG.debug("Applying decorator stack (before default) to: %s %s", args, kwargs)
-        args_kwargs.kwargs = self.apply_to_kwargs_before_default(args_kwargs.kwargs)
-
-        args_kwargs.add_default_values_and_kwargs()
-        self.consolidate_defaults(args_kwargs.defaults)
-
-        LOG.debug("Applying decorator stack to: %s %s", args, kwargs)
-        args_kwargs.kwargs = self.apply_to_kwargs(args_kwargs.kwargs)
-
-        args_kwargs.ensure_positionals_only()
-
-        args, kwargs = args_kwargs.args, args_kwargs.kwargs
-
-        LOG.debug("CALLING func %s %s", args, kwargs)
-        return args, kwargs
-
-
-def normaliser(*args, **kwargs):
-    return Argument("no-name", *args, **kwargs)
-
-
-def safe_to_str(dic):
-    lst = []
-    for k, v in dic.items():
-        try:
-            v = str(v)
-        except:  # noqa: E722
-            v = "..."
-        lst.append(f"{k}={v}")
-    return ",".join(lst)
diff --git a/earthkit/data/sources/ads.py b/earthkit/data/sources/ads.py
new file mode 100644
index 0000000..e298e44
--- /dev/null
+++ b/earthkit/data/sources/ads.py
@@ -0,0 +1,75 @@
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import os
+
+import cdsapi
+import yaml
+
+from .cds import CdsRetriever
+from .prompt import APIKeyPrompt
+
+
+class ADSAPIKeyPrompt(APIKeyPrompt):
+    register_or_sign_in_url = "https://ads.atmosphere.copernicus.eu/"
+    retrieve_api_key_url = "https://ads.atmosphere.copernicus.eu/api-how-to"
+
+    prompts = [
+        dict(
+            name="url",
+            default="https://ads.atmosphere.copernicus.eu/api/v2",
+            title="API url",
+            validate=r"http.?://.*",
+        ),
+        dict(
+            name="key",
+            example="123:abcdef01-0000-1111-2222-0123456789ab",
+            title="API key",
+            hidden=True,
+            validate=r"\d+:[\-0-9a-f]+",
+        ),
+    ]
+
+    rcfile = "~/.adsapirc"
+
+    def save(self, input, file):
+        yaml.dump(input, file, default_flow_style=False)
+
+
+def client():
+    prompt = ADSAPIKeyPrompt()
+    prompt.check()
+
+    path = os.path.expanduser(prompt.rcfile)
+
+    if not os.path.exists(path):
+        prompt.ask_user_and_save()
+
+    with open(path) as f:
+        rc = yaml.safe_load(f.read())
+
+    return cdsapi.Client(**rc)
+
+
+EXTENSIONS = {
+    "grib": ".grib",
+    "netcdf": ".nc",
+}
+
+
+class AdsRetriever(CdsRetriever):
+    sphinxdoc = """
+    AdsRetriever
+    """
+
+    def client(self):
+        return client()
+
+
+source = AdsRetriever
diff --git a/earthkit/data/sources/cds.py b/earthkit/data/sources/cds.py
index e97697c..778cd36 100644
--- a/earthkit/data/sources/cds.py
+++ b/earthkit/data/sources/cds.py
@@ -69,6 +69,9 @@ class CdsRetriever(FileSource):
     CdsRetriever
     """
 
+    def client(self):
+        return client()
+
     def __init__(self, dataset, *args, **kwargs):
         super().__init__()
 
@@ -81,7 +84,7 @@ class CdsRetriever(FileSource):
 
         requests = self.requests(**kwargs)
 
-        client()  # Trigger password prompt before thraeding
+        self.client()  # Trigger password prompt before thraeding
 
         nthreads = min(self.settings("number-of-download-threads"), len(requests))
 
@@ -96,7 +99,7 @@ class CdsRetriever(FileSource):
 
     def _retrieve(self, dataset, request):
         def retrieve(target, args):
-            client().retrieve(args[0], args[1], target)
+            self.client().retrieve(args[0], args[1], target)
 
         return self.cache_file(
             retrieve,
diff --git a/earthkit/data/utils/module_inputs_wrapper.py b/earthkit/data/utils/module_inputs_wrapper.py
new file mode 100644
index 0000000..66e0712
--- /dev/null
+++ b/earthkit/data/utils/module_inputs_wrapper.py
@@ -0,0 +1,197 @@
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+"""
+Module containing methods to transform the inputs of functions based on the function type setting,
+common signitures or mapping defined at call time
+"""
+import inspect
+import types
+import typing as T
+from ast import Module
+from functools import wraps
+
+from earthkit.data import transform
+from earthkit.data.wrappers import Wrapper
+
+try:
+    UNION_TYPES = [T.Union, types.UnionType]
+except AttributeError:
+    # types.UnionType is not in versions of python<3.9
+    UNION_TYPES = [
+        T.Union,
+    ]
+
+EMPTY_TYPES = [inspect._empty]
+
+
+def _ensure_iterable(input_item):
+    """Ensure that an item is iterable"""
+    if not isinstance(input_item, (tuple, list, dict)):
+        return [input_item]
+    return input_item
+
+
+def _ensure_tuple(input_item):
+    """Ensure that an item is a tuple"""
+    if not isinstance(input_item, tuple):
+        return tuple(_ensure_iterable(input_item))
+    return input_item
+
+
+def transform_function_inputs(
+    function: T.Callable,
+    kwarg_types: T.Dict[str, T.Any] = {},
+    convert_types: T.Union[T.Tuple[T.Any], T.Dict[str, T.Tuple[T.Any]]] = (),
+) -> T.Callable:
+    """
+    Transform the inputs to a function to match the requirements.
+    earthkit.data handles the input arg/kwarg format.
+
+    Parameters
+    ----------
+    function : Callable
+        Method to be wrapped
+    kwarg_types : Dict[str: type]
+        Mapping of accepted object types for each arg/kwarg
+    convert_types : Tuple[type]
+        List of data-types to try to convert, this can be useful when the function is versitile and can
+        accept a large number of data-types, hence only a small number of types should be converted.
+
+    Returns
+    -------
+    [type]
+        [description]
+    """
+
+    def _wrapper(kwarg_types, convert_types, *args, **kwargs):
+        kwarg_types = {**kwarg_types}
+        signature = inspect.signature(function)
+        mapping = signature_mapping(signature, kwarg_types)
+
+        # Add args to kwargs for ease of looping:
+        arg_names = []
+        for arg, name in zip(args, signature.parameters):
+            arg_names.append(name)
+            kwargs[name] = arg
+
+        convert_kwargs = [k for k in kwargs if k in mapping]
+        # Only convert some data-types, this can be used to prevent conversion for for functions which
+        #  accept a long-list of formats, e.g. numpy methods can accept xarray, pandas and more
+
+        # Filter for convert_types
+        if convert_types:
+            # Ensure convert_types is a dictionary
+            if not isinstance(convert_types, dict):
+                convert_types = {key: convert_types for key in convert_kwargs}
+
+            convert_kwargs = [
+                k
+                for k in convert_kwargs
+                if isinstance(kwargs[k], _ensure_tuple(convert_types.get(k, ())))
+            ]
+
+        # transform args/kwargs
+        for key in convert_kwargs:
+            value = kwargs[key]
+            kwarg_types = _ensure_iterable(mapping[key])
+            # Transform value if necessary
+            if type(value) not in kwarg_types:
+                for kwarg_type in kwarg_types:
+                    try:
+                        kwargs[key] = transform(value, kwarg_type)
+                    except Exception:
+                        # Transform was not possible, move to next kwarg type.
+                        # If no transform is possible, format is unchanged and we rely on function to raise
+                        # an Error.
+                        continue
+                    break
+
+        # Anything that is still a Wrapper object, expand to native data format:
+        for k, v in [(_k, _v) for _k, _v in kwargs.items() if isinstance(_v, Wrapper)]:
+            try:
+                kwargs[k] = v.data
+            except Exception:
+                pass
+
+        # Extract args from kwargs:
+        args = [kwargs.pop(name) for name in arg_names]
+        return function(*args, **kwargs)
+
+    @wraps(function)
+    def wrapper(*args, _auto_inputs_transform=True, **kwargs):
+        if not _auto_inputs_transform:  # Possibility to escape wrapping at call level
+            return function(*args, **kwargs)
+        return _wrapper(kwarg_types, convert_types, *args, **kwargs)
+
+    return wrapper
+
+
+def signature_mapping(signature, kwarg_types):
+    """
+    Map args and kwargs to object types, using hierarchical selection method:
+    1. Explicitly defined type
+    2. Based on Type setting in function
+    3. Do nothing
+    """
+    mapping = {}
+    for key, parameter in signature.parameters.items():
+        if key in kwarg_types:
+            # 1. Use explicitly defined type
+            kwarg_type = kwarg_types.get(key)
+        elif parameter.annotation not in EMPTY_TYPES:
+            # 2. Use type setting from function
+            kwarg_type = parameter.annotation
+            if T.get_origin(kwarg_type) in UNION_TYPES:
+                # Need to expand union_types to list
+                kwarg_type = T.get_args(kwarg_type)
+        else:
+            # 3. Do nothing, cannot assign None, as None is a valid type
+            continue
+        mapping[key] = kwarg_type
+    return mapping
+
+
+def transform_module_inputs(in_module, **kwargs):
+    """
+    Transform the inputs to all functions in a module.
+
+    Parameters
+    ----------
+    in_module : Module
+        Module containing funcitons which are to be wrapped with transform_function_inputs
+    kwarg_types : Dict[str: type]
+        Mapping of accepted object types for each arg/kwarg
+    convert_types : Tuple[type]
+        List of data-types to try to convert, this can be useful when the function is versitile and can
+        accept a large number of data-types, hence only a small number of types should be converted.
+    kwargs: Any
+        Any other kwargs accepted by transform_function_inputs
+
+    Returns
+    -------
+    Module
+        Version of in_module where all functions are wrapped with transform_modul_inputs
+    """
+    # wrapped_module must be different to original to prevent overriding cached module
+    wrapped_module = Module
+    for name in dir(in_module):
+        func = getattr(in_module, name)
+        # Wrap any functions that are not hidden
+        if not name.startswith("_") and isinstance(func, types.FunctionType):
+            setattr(
+                wrapped_module,
+                name,
+                transform_function_inputs(func, **kwargs),
+            )
+        else:
+            # If not a func, we just copy
+            setattr(wrapped_module, name, func)
+
+    return wrapped_module
diff --git a/earthkit/data/wrappers/ndarray.py b/earthkit/data/wrappers/ndarray.py
index 87be543..054f802 100644
--- a/earthkit/data/wrappers/ndarray.py
+++ b/earthkit/data/wrappers/ndarray.py
@@ -28,6 +28,18 @@ class NumpyNDArrayWrapper(Wrapper):
         """
         return self.data
 
+    def to_xarray(self, **kwargs):
+        """
+        Return an xarray.DataArray representation of the data.
+
+        Returns
+        -------
+        xarray.DataArray
+        """
+        import xarray as xr
+
+        return xr.DataArray(self.data, **kwargs)
+
 
 def wrapper(data, *args, **kwargs):
     import numpy as np


Test patch:
diff --git a/tests/documentation/test_notebooks.py b/tests/documentation/test_notebooks.py
index 3ef29f1..fcaba58 100644
--- a/tests/documentation/test_notebooks.py
+++ b/tests/documentation/test_notebooks.py
@@ -22,7 +22,7 @@ from earthkit.data.testing import MISSING, earthkit_file
 
 EXAMPLES = earthkit_file("docs", "examples")
 
-SKIP = ["fdb.ipynb", "mars.ipynb", "cds.ipynb"]
+SKIP = ["fdb.ipynb", "mars.ipynb", "cds.ipynb", "ads.ipynb"]
 
 
 def notebooks_list():
diff --git a/tests/sources/test_ads.py b/tests/sources/test_ads.py
new file mode 100644
index 0000000..6c5ad02
--- /dev/null
+++ b/tests/sources/test_ads.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import os
+
+import pytest
+
+from earthkit.data import from_source
+
+NO_ADS = not os.path.exists(os.path.expanduser("~/.adsapirc"))
+
+
+@pytest.mark.long_test
+@pytest.mark.download
+@pytest.mark.skipif(NO_ADS, reason="No access to ADS")
+def test_ads_grib_1():
+    s = from_source(
+        "ads",
+        "cams-global-reanalysis-eac4",
+        variable=["particulate_matter_10um", "particulate_matter_1um"],
+        area=[50, -50, 20, 50],
+        date="2012-12-12",
+        time="12:00",
+    )
+    assert len(s) == 2
+
+
+@pytest.mark.long_test
+@pytest.mark.download
+@pytest.mark.skipif(NO_ADS, reason="No access to ADS")
+def test_ads_grib_2():
+    s = from_source(
+        "ads",
+        "cams-global-reanalysis-eac4",
+        variable=["particulate_matter_10um", "particulate_matter_1um"],
+        area=[50, -50, 20, 50],
+        date="2012-12-12",
+        time="12:00",
+        split_on="variable",
+    )
+    assert len(s) == 2
+
+
+@pytest.mark.long_test
+@pytest.mark.download
+@pytest.mark.skipif(NO_ADS, reason="No access to ADS")
+def test_ads_grib_3():
+    s = from_source(
+        "ads",
+        "cams-global-reanalysis-eac4",
+        variable=["particulate_matter_10um", "particulate_matter_1um"],
+        area=[50, -50, 20, 50],
+        date="2012-12-12/to/2012-12-15",
+        time="12:00",
+    )
+    assert len(s) == 8
+
+
+@pytest.mark.long_test
+@pytest.mark.download
+@pytest.mark.skipif(NO_ADS, reason="No access to ADS")
+def test_ads_netcdf():
+    s = from_source(
+        "ads",
+        "cams-global-reanalysis-eac4",
+        variable=["particulate_matter_10um", "particulate_matter_1um"],
+        area=[50, -50, 20, 50],
+        date="2012-12-12",
+        time="12:00",
+        format="netcdf",
+    )
+    assert len(s) == 2
+
+
+if __name__ == "__main__":
+    from earthkit.data.testing import main
+
+    main(__file__)
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/utils/dummy_module.py b/tests/utils/dummy_module.py
new file mode 100644
index 0000000..6165eb1
--- /dev/null
+++ b/tests/utils/dummy_module.py
@@ -0,0 +1,23 @@
+import typing as T
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+
+dummy_attribute_1 = 1
+dummy_attribute_2 = 2
+
+XR_TYPES = (xr.Dataset, xr.DataArray, xr.Variable)
+
+
+def numpy_mean(
+    a: T.Union[
+        np.ndarray, xr.Dataset, xr.DataArray, xr.Variable, pd.DataFrame, pd.Series
+    ],
+    **kwargs
+):
+    return np.mean(a, **kwargs)
+
+
+def xarray_ones_like(dataarray: T.Union[xr.DataArray, xr.Dataset], **kwargs):
+    return xr.ones_like(dataarray, **kwargs)
diff --git a/tests/utils/test_module_inputs_wrapper.py b/tests/utils/test_module_inputs_wrapper.py
new file mode 100644
index 0000000..3df34b8
--- /dev/null
+++ b/tests/utils/test_module_inputs_wrapper.py
@@ -0,0 +1,170 @@
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+
+from earthkit.data import from_object, from_source
+from earthkit.data.readers import Reader
+from earthkit.data.utils import module_inputs_wrapper
+
+from . import dummy_module
+from .dummy_module import XR_TYPES
+
+TEST_NP = np.arange(10)
+TEST_NP2 = np.arange(10)
+
+TEST_DF = pd.DataFrame({"index": TEST_NP, "data": TEST_NP2}).set_index("index")
+
+TEST_DA = xr.DataArray(TEST_NP, name="test")
+TEST_DA2 = xr.DataArray(TEST_NP2, name="test2")
+TEST_DS = TEST_DA.to_dataset()
+TEST_DS["test2"] = TEST_DA2
+
+EK_GRIB_READER = from_source("file", "tests/data/test_single.grib")
+EK_XARRAY_WRAPPER = from_object(TEST_DS)
+EK_NUMPY_WRAPPER = from_object(TEST_NP)
+
+WRAPPED_XR_ONES_LIKE = module_inputs_wrapper.transform_function_inputs(
+    xr.ones_like, kwarg_types={"other": XR_TYPES}
+)
+
+WRAPPED_XR_ONES_LIKE_TYPE_SETTING = module_inputs_wrapper.transform_function_inputs(
+    dummy_module.xarray_ones_like,
+)
+
+WRAPPED_NP_MEAN = module_inputs_wrapper.transform_function_inputs(
+    np.mean,
+    kwarg_types={"a": np.ndarray},
+    convert_types=(
+        Reader
+    ),  # Only convert Earthkit.data.Reader (np.mean can handle xarray and pandas)
+)
+
+WRAPPED_NP_MEAN_TYPE_SETTING = module_inputs_wrapper.transform_function_inputs(
+    dummy_module.numpy_mean,
+)
+
+WRAPPED_DUMMY_MODULE = module_inputs_wrapper.transform_module_inputs(
+    dummy_module,
+)
+
+
+def test_transform_function_inputs_reader_to_xarray():
+    # Check EK GribReader object
+    ek_reader_result = WRAPPED_XR_ONES_LIKE(EK_GRIB_READER)
+    # Will return a DataSet becuase that is first value in kwarg_types
+    assert isinstance(ek_reader_result, xr.Dataset)
+    assert ek_reader_result.equals(xr.ones_like(EK_GRIB_READER.to_xarray()))
+
+
+def test_transform_function_inputs_reader_to_xarray_typesetting():
+    # Check EK GribReader object
+    ek_reader_result = WRAPPED_XR_ONES_LIKE_TYPE_SETTING(EK_GRIB_READER)
+    # Will return a dataarray because that is first value in type-set Union
+    assert isinstance(ek_reader_result, xr.DataArray)
+    assert ek_reader_result.equals(xr.ones_like(EK_GRIB_READER.to_xarray().t2m))
+
+
+def test_transform_module_inputs_reader_to_xarray():
+    # Check EK GribReader object
+    ek_reader_result = WRAPPED_DUMMY_MODULE.xarray_ones_like(EK_GRIB_READER)
+    # Data array because type-setting of function has dataarray first
+    assert isinstance(ek_reader_result, xr.DataArray)
+    assert ek_reader_result.equals(xr.ones_like(EK_GRIB_READER.to_xarray()).t2m)
+
+
+def test_transform_function_inputs_wrapper_to_xarray():
+    # EK XarrayWrapper object
+    ek_wrapper_result = WRAPPED_XR_ONES_LIKE(EK_XARRAY_WRAPPER)
+    assert isinstance(ek_wrapper_result, xr.Dataset)
+    assert ek_wrapper_result.equals(xr.ones_like(EK_XARRAY_WRAPPER.data))
+    # EK NumpyWrapper object
+    ek_wrapper_result = WRAPPED_XR_ONES_LIKE(EK_NUMPY_WRAPPER)
+    assert isinstance(ek_wrapper_result, xr.DataArray)
+    assert ek_wrapper_result.equals(xr.ones_like(TEST_DA))
+
+
+def test_transform_module_inputs_wrapper_to_xarray():
+    # EK XarrayWrapper object
+    ek_wrapper_result = WRAPPED_DUMMY_MODULE.xarray_ones_like(EK_XARRAY_WRAPPER)
+    # Will return a dataarray because that is first value in type-set Union
+    assert isinstance(ek_wrapper_result, xr.DataArray)
+    assert ek_wrapper_result.equals(xr.ones_like(EK_XARRAY_WRAPPER.data.test))
+    # EK NumpyWrapper object
+    ek_wrapper_result = WRAPPED_DUMMY_MODULE.xarray_ones_like(EK_NUMPY_WRAPPER)
+    assert isinstance(ek_wrapper_result, xr.DataArray)
+    assert ek_wrapper_result.equals(xr.ones_like(TEST_DA))
+
+
+def test_transform_function_inputs_reader_to_numpy():
+    # Test with Earthkit.data GribReader object
+    assert WRAPPED_NP_MEAN(EK_GRIB_READER) == np.mean(EK_GRIB_READER.to_numpy())
+    assert isinstance(WRAPPED_NP_MEAN(EK_GRIB_READER), np.float64)
+
+
+def test_transform_function_inputs_reader_to_numpy_typesetting():
+    # Test with Earthkit.data GribReader object
+    result = WRAPPED_NP_MEAN_TYPE_SETTING(EK_GRIB_READER)
+    assert result == np.mean(EK_GRIB_READER.to_numpy())
+    assert isinstance(result, np.float64)
+
+
+def test_transform_module_inputs_reader_to_numpy():
+    # Test with Earthkit.data GribReader object
+    result = WRAPPED_DUMMY_MODULE.numpy_mean(EK_GRIB_READER)
+    assert result == np.mean(EK_GRIB_READER.to_numpy())
+    assert isinstance(result, np.float64)
+
+
+def test_transform_function_inputs_wrapper_to_numpy():
+    # Test with Earthkit.data XarrayWrapper object
+    ek_object_result = WRAPPED_NP_MEAN(EK_XARRAY_WRAPPER)
+    assert ek_object_result == np.mean(TEST_DS)
+    # assert isinstance(ek_object_result, type(EK_XARRAY_WRAPPER.data))
+
+    # Test with Earthkit.data NumpyWrapper object
+    ek_object_result = WRAPPED_NP_MEAN(EK_NUMPY_WRAPPER)
+    assert ek_object_result == np.mean(TEST_NP)
+    assert isinstance(ek_object_result, np.float64)
+
+
+def test_transform_module_inputs_wrapper_to_numpy():
+    # Test with Earthkit.data XarrayWrapper object
+    ek_object_result = WRAPPED_DUMMY_MODULE.numpy_mean(EK_XARRAY_WRAPPER)
+    assert ek_object_result == np.mean(TEST_DS)
+    # TODO: Without definition of convert_types, WRAPPERS are converted to first arguement
+    #       This could be addressed in future versions
+    # assert isinstance(ek_object_result, type(EK_XARRAY_WRAPPER.data))
+
+    # Test with Earthkit.data NumpyWrapper object
+    ek_object_result = WRAPPED_DUMMY_MODULE.numpy_mean(EK_NUMPY_WRAPPER)
+    assert ek_object_result == np.mean(TEST_NP)
+    assert isinstance(ek_object_result, np.float64)
+
+
+def test_transform_function_inputs_xarray_to_numpy():
+    # Test with xarray.DataArray object
+    assert WRAPPED_NP_MEAN(TEST_DA) == np.mean(TEST_NP)
+    assert WRAPPED_NP_MEAN(TEST_DA) == np.mean(TEST_DA)
+    assert isinstance(WRAPPED_NP_MEAN(TEST_DA), xr.DataArray)
+
+
+def test_transform_function_inputs_pandas_to_numpy():
+    # Test with pandas.DataFrame object, axis=0 is to preserve the pandas.DataFrame, this is pandas syntax
+    result = WRAPPED_NP_MEAN(TEST_DF, axis=0)
+    assert result.equals(np.mean(TEST_DF, axis=0))
+    assert isinstance(result, pd.core.series.Series)
+
+    # # Test without axis declartion, pandas>2 returns a scalar mean of the dataframe
+    # # Omitted as breaking change in pandas makes tests unstable
+    # result = WRAPPED_NP_MEAN(TEST_DF)
+    # assert result == np.mean(TEST_DF)
+    # assert isinstance(result, np.float64)