Repository: ecmwf/earthkit-data
Pull request #293 (2024-02-03 16:37:04)

Implement parts for file source for GRIB and BUFR data
This PR enables the `file` source to use parts (byte ranges) in the same way as implemented for the `url` source. At the moment, it works for GRIB and BUFR files.

Documentation was added for both the sources.

Notebook example: https://earthkit-data--293.org.readthedocs.build/en/293/examples/grib_file_parts.html 

Examples:

```python
ds = earthkit.data.from_source("file", "my.grib", parts=(0, 150))
ds = earthkit.data.from_source("file", "my.grib", parts=[(0, 150), (400, 160)])

ds = earthkit.data.from_source("file", [
    ("a.grib", (0, 150)),
    ("b.grib", (240, 120)),
    ("c.grib", None),
    ("d.grib", [(240, 120), (720, 120)])
    ])
```


Patch:
diff --git a/docs/examples.rst b/docs/examples.rst
index ad50dce..fabff3b 100644
--- a/docs/examples.rst
+++ b/docs/examples.rst
@@ -15,7 +15,6 @@ Here is a list of example notebooks to illustrate how to use earthkit-data.
     examples/cds.ipynb
     examples/ecmwf_open_data.ipynb
     examples/fdb.ipynb
-    examples/mars.ipynb
     examples/polytope.ipynb
     examples/wekeo.ipynb
 
@@ -33,6 +32,7 @@ Here is a list of example notebooks to illustrate how to use earthkit-data.
     examples/grib_multi.ipynb
     examples/grib_from_stream.ipynb
     examples/grib_file_pattern.ipynb
+    examples/grib_file_parts.ipynb
     examples/grib_tar.ipynb
     examples/grib_url.ipynb
     examples/grib_url_stream.ipynb
diff --git a/docs/examples/grib_file_parts.ipynb b/docs/examples/grib_file_parts.ipynb
new file mode 100644
index 0000000..fca56ff
--- /dev/null
+++ b/docs/examples/grib_file_parts.ipynb
@@ -0,0 +1,1068 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7b42e29a-7d97-4b49-ab49-db2d73d79311",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "## GRIB file parts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2beb1bac-4968-4f0a-aa1c-8fedbf234b0e",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import earthkit.data"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "a499dce2-5b62-4d7c-b994-ee2020c35468",
+   "metadata": {
+    "editable": true,
+    "raw_mimetype": "text/restructuredtext",
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "This notebook demonstartes how to use only :ref:`parts <parts>` (byte ranges) from GRIB files.\n",
+    "\n",
+    "First we ensure the example files are available."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "313ba1e1-eb22-4bd3-adef-622fce67acb4",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "earthkit.data.download_example_file([\"test.grib\", \"test6.grib\", \"tuv_pl.grib\"])"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "e9ccbcb7-a8d2-4e61-9055-1412cdc372ed",
+   "metadata": {
+    "editable": true,
+    "raw_mimetype": "text/restructuredtext",
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "We load one of the files and inspect the contents with :py:meth:`~data.readers.grib.index.GribFieldList.ls`. By using the \"offset\" key we can get the byte positions where each message starts within the file. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "27820099-8b9c-4c6c-9ee0-a72ef4224dac",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "      <th>offset</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>u</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "      <td>240.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>v</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "      <td>480.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>850</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "      <td>720.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>u</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>850</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "      <td>960.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>v</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>850</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "      <td>1200.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName    typeOfLevel  level  dataDate  dataTime stepRange  \\\n",
+       "0   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "1   ecmf         u  isobaricInhPa   1000  20180801      1200         0   \n",
+       "2   ecmf         v  isobaricInhPa   1000  20180801      1200         0   \n",
+       "3   ecmf         t  isobaricInhPa    850  20180801      1200         0   \n",
+       "4   ecmf         u  isobaricInhPa    850  20180801      1200         0   \n",
+       "5   ecmf         v  isobaricInhPa    850  20180801      1200         0   \n",
+       "\n",
+       "  dataType  number    gridType  offset  \n",
+       "0       an       0  regular_ll     0.0  \n",
+       "1       an       0  regular_ll   240.0  \n",
+       "2       an       0  regular_ll   480.0  \n",
+       "3       an       0  regular_ll   720.0  \n",
+       "4       an       0  regular_ll   960.0  \n",
+       "5       an       0  regular_ll  1200.0  "
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds = earthkit.data.from_source(\"file\", \"test6.grib\")\n",
+    "ds.ls(extra_keys=\"offset\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f348d7e7-d864-4e23-bb1b-adf6f1143c9c",
+   "metadata": {},
+   "source": [
+    "### Single files"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "e8346eb4-804f-4093-bda8-80acec9873ae",
+   "metadata": {
+    "editable": true,
+    "raw_mimetype": "text/restructuredtext",
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "The **parts** option of :ref:`from_source() <data-sources-file>` specifies the **byte range(s)** we want to read from a file. A single :ref:`part <parts>` is a tuple or list in the following format: *(offset, length)*. \n",
+    "\n",
+    "Using the offsets from the example above we can specify the ref:`part <parts>` for the fist message."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "5a048a9f-7ea2-4613-997f-4d7464a7db98",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName    typeOfLevel  level  dataDate  dataTime stepRange  \\\n",
+       "0   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "\n",
+       "  dataType  number    gridType  \n",
+       "0       an       0  regular_ll  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds = earthkit.data.from_source(\"file\", \"test6.grib\", parts=(0, 240))\n",
+    "ds.ls()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b40e2a4d-d000-4a3f-b756-0c8780e6bf09",
+   "metadata": {},
+   "source": [
+    "The call above can also be written as:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "661e7ebb-13f7-4db6-8e87-ee59848c47b2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName    typeOfLevel  level  dataDate  dataTime stepRange  \\\n",
+       "0   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "\n",
+       "  dataType  number    gridType  \n",
+       "0       an       0  regular_ll  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds = earthkit.data.from_source(\"file\", \"test6.grib\", parts=[(0, 240)])\n",
+    "ds.ls()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5c8001e3-5413-4c8f-ac4e-ff3a7d168a43",
+   "metadata": {},
+   "source": [
+    "A part can go over a message boundary. Here bytes 240-244 belong to the second message, which is not read because not all of its bytes are specified."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "cdbd5356-8696-4e73-9d1d-1996667f1792",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName    typeOfLevel  level  dataDate  dataTime stepRange  \\\n",
+       "0   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "\n",
+       "  dataType  number    gridType  \n",
+       "0       an       0  regular_ll  "
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds = earthkit.data.from_source(\"file\", \"test6.grib\", parts=[(0, 245)])\n",
+    "ds.ls()"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "d2c2aa43-802c-4739-b7da-92c5fc22b7b2",
+   "metadata": {
+    "editable": true,
+    "raw_mimetype": "text/restructuredtext",
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "Multiple ref:`parts <parts>` can be used."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "53c60306-ff16-40ce-9922-6552bab3244f",
+   "metadata": {
+    "editable": true,
+    "scrolled": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>v</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>850</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName    typeOfLevel  level  dataDate  dataTime stepRange  \\\n",
+       "0   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "1   ecmf         v  isobaricInhPa   1000  20180801      1200         0   \n",
+       "2   ecmf         t  isobaricInhPa    850  20180801      1200         0   \n",
+       "\n",
+       "  dataType  number    gridType  \n",
+       "0       an       0  regular_ll  \n",
+       "1       an       0  regular_ll  \n",
+       "2       an       0  regular_ll  "
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds = earthkit.data.from_source(\"file\", \"test6.grib\", parts=[(0, 240), (480, 480)])\n",
+    "ds.ls()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a9b26883-6f17-4933-90ad-d2342ac1a26b",
+   "metadata": {},
+   "source": [
+    "Parts cannot overlap."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "01e41b9b-6900-4057-bbe8-958ba2b021bf",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Offsets and lengths must be in order, and not overlapping: offset=220, end of previous part=240\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    ds = earthkit.data.from_source(\"file\", \"test6.grib\", parts=[(0, 240), (220, 240)])\n",
+    "except Exception as e:\n",
+    "    print(e)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f090b522-b8dc-4d54-8b49-3ea2a78ed16f",
+   "metadata": {},
+   "source": [
+    "### Multiple files"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "28957198-65d5-480d-809d-26a1aac27a30",
+   "metadata": {
+    "editable": true,
+    "raw_mimetype": "text/restructuredtext",
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "When using multiple files we can specify the ref:`part <parts>` for each file with the following syntax:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "4b1a35fe-db43-4e33-b00e-f55c5c2332d1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>2t</td>\n",
+       "      <td>surface</td>\n",
+       "      <td>0</td>\n",
+       "      <td>20200513</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>v</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName    typeOfLevel  level  dataDate  dataTime stepRange  \\\n",
+       "0   ecmf        2t        surface      0  20200513      1200         0   \n",
+       "1   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "2   ecmf         v  isobaricInhPa   1000  20180801      1200         0   \n",
+       "\n",
+       "  dataType  number    gridType  \n",
+       "0       an       0  regular_ll  \n",
+       "1       an       0  regular_ll  \n",
+       "2       an       0  regular_ll  "
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds = earthkit.data.from_source(\"file\", [\n",
+    "                               [\"test.grib\", (0,526)], \n",
+    "                               [\"test6.grib\", [(0, 240), (480, 240)]]\n",
+    "                              ])\n",
+    "ds.ls()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b77c770f-0f3c-48f4-afe4-bcac35d424a0",
+   "metadata": {},
+   "source": [
+    "When a part is None for a given file the whole file will be used."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "ba09e8ea-19c1-4763-b603-acedad380ed5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>2t</td>\n",
+       "      <td>surface</td>\n",
+       "      <td>0</td>\n",
+       "      <td>20200513</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>msl</td>\n",
+       "      <td>surface</td>\n",
+       "      <td>0</td>\n",
+       "      <td>20200513</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>v</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName    typeOfLevel  level  dataDate  dataTime stepRange  \\\n",
+       "0   ecmf        2t        surface      0  20200513      1200         0   \n",
+       "1   ecmf       msl        surface      0  20200513      1200         0   \n",
+       "2   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "3   ecmf         v  isobaricInhPa   1000  20180801      1200         0   \n",
+       "\n",
+       "  dataType  number    gridType  \n",
+       "0       an       0  regular_ll  \n",
+       "1       an       0  regular_ll  \n",
+       "2       an       0  regular_ll  \n",
+       "3       an       0  regular_ll  "
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds = earthkit.data.from_source(\"file\", [\n",
+    "                               [\"test.grib\", None], \n",
+    "                               [\"test6.grib\", [(0,240), (480, 240)]]\n",
+    "                              ])\n",
+    "ds.ls()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "77361bdd-5294-4a0b-8b9e-0a283483ff80",
+   "metadata": {},
+   "source": [
+    "The **parts** kwarg can still be used for multiple files; in this case it will be applied to each of them one by one."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "007d715f-291a-4f17-b252-7cf717ad7426",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>centre</th>\n",
+       "      <th>shortName</th>\n",
+       "      <th>typeOfLevel</th>\n",
+       "      <th>level</th>\n",
+       "      <th>dataDate</th>\n",
+       "      <th>dataTime</th>\n",
+       "      <th>stepRange</th>\n",
+       "      <th>dataType</th>\n",
+       "      <th>number</th>\n",
+       "      <th>gridType</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ecmf</td>\n",
+       "      <td>t</td>\n",
+       "      <td>isobaricInhPa</td>\n",
+       "      <td>1000</td>\n",
+       "      <td>20180801</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>0</td>\n",
+       "      <td>an</td>\n",
+       "      <td>0</td>\n",
+       "      <td>regular_ll</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  centre shortName    typeOfLevel  level  dataDate  dataTime stepRange  \\\n",
+       "0   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "1   ecmf         t  isobaricInhPa   1000  20180801      1200         0   \n",
+       "\n",
+       "  dataType  number    gridType  \n",
+       "0       an       0  regular_ll  \n",
+       "1       an       0  regular_ll  "
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds = earthkit.data.from_source(\"file\", [\"test6.grib\", \"tuv_pl.grib\"], parts=(0,240))\n",
+    "ds.ls()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "53be8b67-8260-456b-bc58-12aa4a3380d3",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "dev_ecc",
+   "language": "python",
+   "name": "dev_ecc"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/docs/guide/index.rst b/docs/guide/index.rst
index 7723658..feed122 100644
--- a/docs/guide/index.rst
+++ b/docs/guide/index.rst
@@ -10,3 +10,4 @@ User guide
    data_format/index.rst
    settings
    caching
+   misc/index.rst
diff --git a/docs/guide/misc/index.rst b/docs/guide/misc/index.rst
new file mode 100644
index 0000000..47200b9
--- /dev/null
+++ b/docs/guide/misc/index.rst
@@ -0,0 +1,8 @@
+Miscellaneous
+===============
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+
+   *
diff --git a/docs/guide/misc/parts.rst b/docs/guide/misc/parts.rst
new file mode 100644
index 0000000..d372215
--- /dev/null
+++ b/docs/guide/misc/parts.rst
@@ -0,0 +1,60 @@
+.. _parts:
+
+Using parts for file and url sources
+====================================
+
+For the :ref:`data-sources-file` and :ref:`data-sources-url` sources we can specify the parts (byte ranges) we want to read from the given input.
+
+A **single part** is a pair (list or tuple) in the following format::
+
+    (offset, length)
+
+where:
+
+    - offset: the start byte position
+    - length: the number of bytes to be read from the offset
+
+**Multiple parts** can be defined as a list or tuple of single parts. These have to be specified in an ascending offset order, no overlapping is allowed.
+
+
+Using the parts kwarg
+----------------------
+
+For both sources we can use the ``parts`` kwarg to define the parts for all the ``path``\s or ``urls``\ s. A few examples:
+
+.. code-block:: python
+
+    # file
+    ds = earthkit.data.from_source("file", "my.grib", parts=(0, 150))
+    ds = earthkit.data.from_source("file", "my.grib", parts=[(0, 150), (400, 160)])
+
+    # url
+    ds = earthkit.data.from_source("url", "http://my-host/my.grib", parts=(0, 150))
+    ds = earthkit.data.from_source(
+        "url", "http://my-host/my.grib", parts=[(0, 150), (400, 160)]
+    )
+
+
+Specifying the parts for each file
+-------------------------------------
+
+If we have multiple paths/urls and want to specify different parts for each we need to add the parts to the  ``path`` or ``urls`` arguments.
+
+.. code-block:: python
+
+    # file
+    ds = earthkit.data.from_source(
+        "file",
+        [
+            ("a.grib", (0, 150)),
+            ("b.grib", (240, 120)),
+            ("c.grib", None),
+            ("d.grib", [(240, 120), (720, 120)]),
+        ],
+    )
+
+When we use this mode the ``parts`` kwarg cannot be used!
+
+Examples:
+
+    - :ref:`/examples/grib_file_parts.ipynb`
diff --git a/docs/guide/split_on.rst b/docs/guide/misc/split_on.rst
similarity index 100%
rename from docs/guide/split_on.rst
rename to docs/guide/misc/split_on.rst
diff --git a/docs/guide/sources.rst b/docs/guide/sources.rst
index 23e28f2..06b2263 100644
--- a/docs/guide/sources.rst
+++ b/docs/guide/sources.rst
@@ -60,17 +60,19 @@ We can get data from a given source by using :func:`from_source`:
 file
 ----
 
-.. py:function:: from_source("file", path, expand_user=True, expand_vars=False, unix_glob=True, recursive_glob=True)
+.. py:function:: from_source("file", path, expand_user=True, expand_vars=False, unix_glob=True, recursive_glob=True, parts=None)
   :noindex:
 
-  The simplest source is ``file`` that can access a local file/list of files.
+  The simplest source is ``file``, which can access a local file/list of files.
 
-  :param path: input path(s)
-  :type path: str, list
+  :param path: input path(s). Each path can contain the :ref:`parts <parts>` defining the byte ranges to read.
+  :type path: str, list, tuple
   :param bool expand_user: replace the leading ~ or ~user in ``path`` by that user's home directory. See ``os.path.expanduser``
   :param bool expand_vars:  expand shell environment variables in ``path``. See ``os.path.expandpath``
   :param bool unix_glob: allow UNIX globbing in ``path``
   :param bool recursive_glob: allow recursive scanning of directories. Only used when ``uxix_glob`` is True
+  :param parts: the :ref:`parts <parts>` to read from the file(s) specified by ``path``. Cannot be used when ``path`` already defines the :ref:`parts <parts>`.
+  :type parts: pair, list or tuple of pairs, None
 
   *earthkit-data* will inspect the content of the files to check for any of the
   supported :ref:`data formats <data-format>`.
@@ -94,10 +96,33 @@ file
       ds = earthkit.data.from_source("file", "path/to/dir")
 
 
+  The following examples using parts:
+
+  .. code:: python
+
+      import earthkit.data
+
+      # reading only certain parts (byte ranges) from a single file
+      ds = earthkit.data.from_source("file", "my.grib", parts=[(0, 150), (400, 160)])
+
+      # reading only certain parts (byte ranges) from multiple files
+      ds = earthkit.data.from_source(
+          "file",
+          [
+              ("a.grib", (0, 150)),
+              ("b.grib", (240, 120)),
+              ("c.grib", None),
+              ("d.grib", [(240, 120), (720, 120)]),
+          ],
+      )
+
+
+
   Further examples:
 
     - :ref:`/examples/grib_overview.ipynb`
     - :ref:`/examples/grib_multi.ipynb`
+    - :ref:`/examples/grib_file_parts.ipynb`
     - :ref:`/examples/bufr_temp.ipynb`
     - :ref:`/examples/netcdf.ipynb`
     - :ref:`/examples/odb.ipynb`
@@ -144,14 +169,16 @@ file-pattern
 url
 ---
 
-.. py:function:: from_source("url", url, unpack=True, stream=False, batch_size=1, group_by=None)
+.. py:function:: from_source("url", url, unpack=True, parts=None, stream=False, batch_size=1, group_by=None)
   :noindex:
 
   The ``url`` source will download the data from the address specified and store it in the :ref:`cache <caching>`. The supported data formats are the same as for the :ref:`file <data-sources-file>` data source above.
 
-  :param url: the URL to download
+  :param url: the URL(s) to download. Each URL can contain the :ref:`parts <parts>` defining the byte ranges to read.
   :type url: str
   :param bool unpack: for archive formats such as ``.zip``, ``.tar``, ``.tar.gz``, etc, *earthkit-data* will attempt to open it and extract any usable file. To keep the downloaded file as is use ``unpack=False``
+  :param parts: the :ref:`parts <parts>` to read from the resource(s) specified by ``url``. Cannot be used when ``url`` already defines the :ref:`parts <parts>`.
+  :type parts: pair, list or tuple of pairs, None
   :param bool stream: when it is ``True`` the data is read as a stream. Otherwise the data is retrieved into a file and stored in the :ref:`cache <caching>`. This option only works for GRIB data. No archive formats supported (``unpack`` is ignored). ``stream`` only works for ``http`` and ``https`` URLs.
   :param int batch_size: used when ``stream=True`` and ``group_by`` is unset. It defines how many GRIB messages are consumed from the stream and kept in memory at a time. For details see :ref:`stream source <data-sources-stream>`.
   :param group_by: used when ``stream=True`` and can specify one or more metadata keys to control how GRIB messages are read from the stream. For details see :ref:`stream source <data-sources-stream>`.
@@ -165,8 +192,25 @@ url
       ...     "url",
       ...     "https://get.ecmwf.int/repository/test-data/earthkit-data/examples/test4.grib",
       ... )
-      >>> len(ds)
-      4
+      >>> ds.ls()
+        centre shortName    typeOfLevel  level  dataDate  dataTime stepRange dataType  number    gridType
+      0   ecmf         t  isobaricInhPa    500  20070101      1200         0       an       0  regular_ll
+      1   ecmf         z  isobaricInhPa    500  20070101      1200         0       an       0  regular_ll
+      2   ecmf         t  isobaricInhPa    850  20070101      1200         0       an       0  regular_ll
+      3   ecmf         z  isobaricInhPa    850  20070101      1200         0       an       0  regular_ll
+
+  .. code-block:: python
+
+      >>> import earthkit.data
+      >>> ds = earthkit.data.from_source(
+      ...     "url",
+      ...     "https://get.ecmwf.int/repository/test-data/earthkit-data/examples/test4.grib",
+      ...     parts=[(0, 130428), (260856, 130428)],
+      ... )
+      >>> ds.ls()
+        centre shortName    typeOfLevel  level  dataDate  dataTime stepRange dataType  number    gridType
+      0   ecmf         t  isobaricInhPa    500  20070101      1200         0       an       0  regular_ll
+      1   ecmf         t  isobaricInhPa    850  20070101      1200         0       an       0  regular_ll
 
   Further examples:
 
diff --git a/earthkit/data/readers/__init__.py b/earthkit/data/readers/__init__.py
index 8bd0944..e4e6262 100644
--- a/earthkit/data/readers/__init__.py
+++ b/earthkit/data/readers/__init__.py
@@ -24,7 +24,7 @@ class ReaderMeta(type(Base), type(os.PathLike)):
 
 
 class Reader(Base, os.PathLike, metaclass=ReaderMeta):
-    appendable = False  # Set to True if the data can be appened to and existing file
+    appendable = False  # Set to True if the data can be appended to and existing file
     binary = True
 
     def __init__(self, source, path):
@@ -83,6 +83,9 @@ class Reader(Base, os.PathLike, metaclass=ReaderMeta):
         LOG.warning(f"index-content(): Ignoring {self.path}")
         return []
 
+    def ranges(self):
+        self.source._kw
+
 
 _READERS = {}
 
@@ -140,7 +143,7 @@ def _unknown(method_name, source, path_or_data, **kwargs):
     return unknowns[method_name](source, path_or_data, **kwargs)
 
 
-def reader(source, path, content_type=None):
+def reader(source, path, **kwargs):
     """Create the reader for a file/directory specified by path"""
     assert isinstance(path, str), source
 
@@ -175,7 +178,7 @@ def reader(source, path, content_type=None):
         source,
         path,
         magic=magic,
-        content_type=content_type,
+        **kwargs,
     )
 
 
diff --git a/earthkit/data/readers/bufr/__init__.py b/earthkit/data/readers/bufr/__init__.py
index b2dbe71..eea2125 100644
--- a/earthkit/data/readers/bufr/__init__.py
+++ b/earthkit/data/readers/bufr/__init__.py
@@ -21,8 +21,8 @@ def _match_magic(magic, deeper_check):
     return False
 
 
-def reader(source, path, *, magic=None, deeper_check=False, **kwargs):
+def reader(source, path, *, magic=None, deeper_check=False, parts=None, **kwargs):
     if _match_magic(magic, deeper_check):
         from .bufr import BUFRReader
 
-        return BUFRReader(source, path)
+        return BUFRReader(source, path, parts=parts)
diff --git a/earthkit/data/readers/bufr/bufr.py b/earthkit/data/readers/bufr/bufr.py
index 87ae8b8..57ff086 100644
--- a/earthkit/data/readers/bufr/bufr.py
+++ b/earthkit/data/readers/bufr/bufr.py
@@ -45,39 +45,38 @@ BUFR_LS_KEYS = {
 
 
 class BufrCodesMessagePositionIndex(CodesMessagePositionIndex):
+    MAGIC = b"BUFR"
+
     # This does not belong here, should be in the C library
-    def _get_message_positions(self, path):
-        fd = os.open(path, os.O_RDONLY)
-        try:
+    def _get_message_positions_part(self, fd, part):
+        assert part is not None
+        assert len(part) == 2
 
-            def get(count):
-                buf = os.read(fd, count)
-                assert len(buf) == count
-                return int.from_bytes(
-                    buf,
-                    byteorder="big",
-                    signed=False,
-                )
-
-            offset = 0
-            while True:
-                code = os.read(fd, 4)
-                if len(code) < 4:
-                    break
+        offset = part[0]
+        end_pos = part[0] + part[1] if part[1] > 0 else -1
+
+        if os.lseek(fd, offset, os.SEEK_SET) != offset:
+            return
+
+        while True:
+            code = os.read(fd, 4)
+            if len(code) < 4:
+                break
+
+            if code != self.MAGIC:
+                offset = os.lseek(fd, offset + 1, os.SEEK_SET)
+                continue
 
-                if code != b"BUFR":
-                    offset = os.lseek(fd, offset + 1, os.SEEK_SET)
-                    continue
+            length = self._get_bytes(fd, 3)
+            edition = self._get_bytes(fd, 1)
 
-                length = get(3)
-                edition = get(1)
+            if end_pos > 0 and offset + length > end_pos:
+                return
 
-                if edition in [3, 4]:
-                    yield offset, length
-                    offset = os.lseek(fd, offset + length, os.SEEK_SET)
+            if edition in [3, 4]:
+                yield offset, length
 
-        finally:
-            os.close(fd)
+            offset = os.lseek(fd, offset + length, os.SEEK_SET)
 
 
 class BUFRCodesHandle(CodesHandle):
@@ -629,10 +628,6 @@ class MultiBUFRList(BUFRList, MultiIndex):
 
 
 class BUFRInFiles(BUFRList):
-    # Remote BUFRLists (with urls) are also here,
-    # as the actual fieldlist is accessed on a file in cache.
-    # This class changes the interface (__getitem__ and __len__)
-    # into the interface (part and number_of_parts).
     def _getitem(self, n):
         if isinstance(n, int):
             part = self.part(n if n >= 0 else len(self) + n)
@@ -651,14 +646,17 @@ class BUFRInFiles(BUFRList):
 
 
 class BUFRInOneFile(BUFRInFiles):
-    def __init__(self, path):
+    def __init__(self, path, parts=None):
         self.path = path
+        self._file_parts = parts
         self.__positions = None
 
     @property
     def _positions(self):
         if self.__positions is None:
-            self.__positions = BufrCodesMessagePositionIndex(self.path)
+            self.__positions = BufrCodesMessagePositionIndex(
+                self.path, parts=self._file_parts
+            )
         return self.__positions
 
     def part(self, n):
@@ -673,9 +671,9 @@ class BUFRReader(BUFRInOneFile, Reader):
 
     appendable = True  # BUFR messages can be added to the same file
 
-    def __init__(self, source, path):
+    def __init__(self, source, path, parts=None):
         Reader.__init__(self, source, path)
-        BUFRInOneFile.__init__(self, path)
+        BUFRInOneFile.__init__(self, path, parts=parts)
 
     def __repr__(self):
         return "BUFRReader(%s)" % (self.path,)
diff --git a/earthkit/data/readers/directory.py b/earthkit/data/readers/directory.py
index 6528cbb..226de9d 100644
--- a/earthkit/data/readers/directory.py
+++ b/earthkit/data/readers/directory.py
@@ -91,5 +91,5 @@ class DirectoryReader(Reader):
 
 
 def reader(source, path, *, magic=None, deeper_check=False, **kwargs):
-    if magic is None or os.path.isdir(path):
+    if magic is None and os.path.isdir(path):
         return DirectoryReader(source, path)
diff --git a/earthkit/data/readers/geojson.py b/earthkit/data/readers/geojson.py
index 7e8487b..fabe2e5 100644
--- a/earthkit/data/readers/geojson.py
+++ b/earthkit/data/readers/geojson.py
@@ -107,5 +107,5 @@ def reader(source, path, *, magic=None, deeper_check=False, **kwargs):
 
     geojson_extensions = ["geojson"]
     geojson_mimetypes = ["application/geo+json"]
-    if magic is None or ext in geojson_extensions or kind in geojson_mimetypes:
+    if ext in geojson_extensions or kind in geojson_mimetypes:
         return GeojsonReader(source, path)
diff --git a/earthkit/data/readers/grib/__init__.py b/earthkit/data/readers/grib/__init__.py
index c4e6586..5c8c636 100644
--- a/earthkit/data/readers/grib/__init__.py
+++ b/earthkit/data/readers/grib/__init__.py
@@ -27,11 +27,11 @@ def _is_default(magic, content_type):
     )
 
 
-def reader(source, path, *, magic=None, deeper_check=False, **kwargs):
+def reader(source, path, *, magic=None, deeper_check=False, parts=None, **kwargs):
     if _match_magic(magic, deeper_check):
         from .reader import GRIBReader
 
-        return GRIBReader(source, path)
+        return GRIBReader(source, path, parts=parts)
 
 
 def memory_reader(source, buffer, *, magic=None, deeper_check=False, **kwargs):
@@ -49,7 +49,7 @@ def stream_reader(
     deeper_check=False,
     content_type=None,
     memory=False,
-    **kwargs
+    **kwargs,
 ):
     if _is_default(magic, content_type) or _match_magic(magic, deeper_check):
         from .memory import GribFieldListInMemory, GribStreamReader
diff --git a/earthkit/data/readers/grib/codes.py b/earthkit/data/readers/grib/codes.py
index a6fa002..62a3b5c 100644
--- a/earthkit/data/readers/grib/codes.py
+++ b/earthkit/data/readers/grib/codes.py
@@ -84,64 +84,62 @@ LONGITUDE_ACCESSOR = GribCodesLongitudeAccessor()
 
 
 class GribCodesMessagePositionIndex(CodesMessagePositionIndex):
+    MAGIC = b"GRIB"
+
     # This does not belong here, should be in the C library
-    def _get_message_positions(self, path):
-        fd = os.open(path, os.O_RDONLY)
-        try:
+    def _get_message_positions_part(self, fd, part):
+        assert part is not None
+        assert len(part) == 2
+
+        offset = part[0]
+        end_pos = part[0] + part[1] if part[1] > 0 else -1
+
+        if os.lseek(fd, offset, os.SEEK_SET) != offset:
+            return
+
+        while True:
+            code = os.read(fd, 4)
+            if len(code) < 4:
+                break
+
+            if code != self.MAGIC:
+                offset = os.lseek(fd, offset + 1, os.SEEK_SET)
+                continue
+
+            length = self._get_bytes(fd, 3)
+            edition = self._get_bytes(fd, 1)
+
+            if edition == 1:
+                if length & 0x800000:
+                    sec1len = self._get_bytes(fd, 3)
+                    os.lseek(fd, 4, os.SEEK_CUR)
+                    flags = self._get_bytes(1)
+                    os.lseek(fd, sec1len - 8, os.SEEK_CUR)
+
+                    if flags & (1 << 7):
+                        sec2len = self._get_bytes(fd, 3)
+                        os.lseek(fd, sec2len - 3, os.SEEK_CUR)
+
+                    if flags & (1 << 6):
+                        sec3len = self._get_bytes(fd, 3)
+                        os.lseek(fd, sec3len - 3, os.SEEK_CUR)
+
+                    sec4len = self._get_bytes(fd, 3)
+
+                    if sec4len < 120:
+                        length &= 0x7FFFFF
+                        length *= 120
+                        length -= sec4len
+                        length += 4
+
+            if edition == 2:
+                length = self._get_bytes(fd, 8)
+
+            if end_pos > 0 and offset + length > end_pos:
+                return
 
-            def get(count):
-                buf = os.read(fd, count)
-                assert len(buf) == count
-                return int.from_bytes(
-                    buf,
-                    byteorder="big",
-                    signed=False,
-                )
-
-            offset = 0
-            while True:
-                code = os.read(fd, 4)
-                if len(code) < 4:
-                    break
-
-                if code != b"GRIB":
-                    offset = os.lseek(fd, offset + 1, os.SEEK_SET)
-                    continue
-
-                length = get(3)
-                edition = get(1)
-
-                if edition == 1:
-                    if length & 0x800000:
-                        sec1len = get(3)
-                        os.lseek(fd, 4, os.SEEK_CUR)
-                        flags = get(1)
-                        os.lseek(fd, sec1len - 8, os.SEEK_CUR)
-
-                        if flags & (1 << 7):
-                            sec2len = get(3)
-                            os.lseek(fd, sec2len - 3, os.SEEK_CUR)
-
-                        if flags & (1 << 6):
-                            sec3len = get(3)
-                            os.lseek(fd, sec3len - 3, os.SEEK_CUR)
-
-                        sec4len = get(3)
-
-                        if sec4len < 120:
-                            length &= 0x7FFFFF
-                            length *= 120
-                            length -= sec4len
-                            length += 4
-
-                if edition == 2:
-                    length = get(8)
-
-                yield offset, length
-                offset = os.lseek(fd, offset + length, os.SEEK_SET)
-
-        finally:
-            os.close(fd)
+            yield offset, length
+            offset = os.lseek(fd, offset + length, os.SEEK_SET)
 
 
 class GribCodesHandle(CodesHandle):
diff --git a/earthkit/data/readers/grib/index/__init__.py b/earthkit/data/readers/grib/index/__init__.py
index 4b7cf6c..bdb9128 100644
--- a/earthkit/data/readers/grib/index/__init__.py
+++ b/earthkit/data/readers/grib/index/__init__.py
@@ -191,10 +191,6 @@ class GribMultiFieldList(GribFieldList, MultiIndex):
 
 
 class GribFieldListInFiles(GribFieldList):
-    # Remote FieldLists (with urls) are also here,
-    # as the actual fieldlist is accessed on a file in cache.
-    # This class changes the interface (__getitem__ and __len__)
-    # into the interface (part and number_of_parts).
     def _getitem(self, n):
         if isinstance(n, int):
             part = self.part(n if n >= 0 else len(self) + n)
diff --git a/earthkit/data/readers/grib/index/file.py b/earthkit/data/readers/grib/index/file.py
index 93549a3..0ee44cb 100644
--- a/earthkit/data/readers/grib/index/file.py
+++ b/earthkit/data/readers/grib/index/file.py
@@ -25,21 +25,24 @@ class GribFieldListInOneFile(GribFieldListInFiles):
     def availability_path(self):
         return os.path.join(self.path, ".availability.pickle")
 
-    def __init__(self, path, **kwargs):
+    def __init__(self, path, parts=None, **kwargs):
         assert isinstance(path, str), path
 
         self.path = path
+        self._file_parts = parts
         self.__positions = None
         super().__init__(**kwargs)
 
     @property
     def _positions(self):
         if self.__positions is None:
-            self.__positions = GribCodesMessagePositionIndex(self.path)
+            self.__positions = GribCodesMessagePositionIndex(
+                self.path, self._file_parts
+            )
         return self.__positions
 
     def part(self, n):
         return Part(self.path, self._positions.offsets[n], self._positions.lengths[n])
 
     def number_of_parts(self):
-        return len(self._positions.offsets)
+        return len(self._positions)
diff --git a/earthkit/data/readers/grib/reader.py b/earthkit/data/readers/grib/reader.py
index 0a74313..2fa973e 100644
--- a/earthkit/data/readers/grib/reader.py
+++ b/earthkit/data/readers/grib/reader.py
@@ -19,9 +19,9 @@ LOG = logging.getLogger(__name__)
 class GRIBReader(GribFieldListInOneFile, Reader):
     appendable = True  # GRIB messages can be added to the same file
 
-    def __init__(self, source, path):
+    def __init__(self, source, path, parts=None):
         Reader.__init__(self, source, path)
-        GribFieldListInOneFile.__init__(self, path)
+        GribFieldListInOneFile.__init__(self, path, parts=parts)
 
     def __repr__(self):
         return "GRIBReader(%s)" % (self.path,)
diff --git a/earthkit/data/sources/file.py b/earthkit/data/sources/file.py
index 082b5d7..c080c0b 100644
--- a/earthkit/data/sources/file.py
+++ b/earthkit/data/sources/file.py
@@ -15,6 +15,7 @@ import os
 from earthkit.data import from_source
 from earthkit.data.core.caching import CACHE
 from earthkit.data.readers import reader
+from earthkit.data.utils.parts import check_urls_and_parts, ensure_urls_and_parts
 
 from . import Source
 
@@ -33,11 +34,15 @@ class FileSource(Source, os.PathLike, metaclass=FileSourceMeta):
     _reader_ = None
     content_type = None
 
-    def __init__(self, path=None, filter=None, merger=None, **kwargs):
+    def __init__(self, path=None, filter=None, merger=None, parts=None, **kwargs):
         Source.__init__(self, **kwargs)
-        self.path = path
         self.filter = filter
         self.merger = merger
+        self.path, self.parts = self._paths_and_parts(path, parts)
+
+        if self._kwargs.get("indexing", False):
+            if self.parts is not None and any(x is not None for x in self.parts):
+                raise ValueError("Cannot specify parts when indexing is enabled!")
 
     def mutate(self):
         if isinstance(self.path, (list, tuple)):
@@ -46,7 +51,10 @@ class FileSource(Source, os.PathLike, metaclass=FileSourceMeta):
             else:
                 return from_source(
                     "multi",
-                    [from_source("file", p, **self._kwargs) for p in self.path],
+                    [
+                        from_source("file", p, parts=part, **self._kwargs)
+                        for p, part in zip(self.path, self.parts)
+                    ],
                     filter=self.filter,
                     merger=self.merger,
                 )
@@ -80,7 +88,9 @@ class FileSource(Source, os.PathLike, metaclass=FileSourceMeta):
     @property
     def _reader(self):
         if self._reader_ is None:
-            self._reader_ = reader(self, self.path, content_type=self.content_type)
+            self._reader_ = reader(
+                self, self.path, content_type=self.content_type, parts=self.parts
+            )
         return self._reader_
 
     def __iter__(self):
@@ -177,6 +187,45 @@ class FileSource(Source, os.PathLike, metaclass=FileSourceMeta):
     def statistics(self, **kwargs):
         return self._reader.statistics(**kwargs)
 
+    @staticmethod
+    def _paths_and_parts(paths, parts):
+        """Preprocess paths and parts.
+
+        Parameters
+        ----------
+        paths: str or list/tuple
+            The path(s). When it is a sequence either each
+            item is a path (str), or a pair of a path and :ref:`parts <parts>`.
+        parts: part,list/tuple of parts or None.
+            The :ref:`parts <parts>`.
+
+        Returns
+        -------
+        str or list of str
+            The path or paths.
+        SimplePart, list or tuple, None
+            The parts (one for each path). A part can be a single
+            SimplePart, a list/tuple of SimpleParts or None.
+
+        """
+        if parts is None:
+            if isinstance(paths, str):
+                return paths, None
+            elif isinstance(paths, (list, tuple)) and all(
+                isinstance(p, str) for p in paths
+            ):
+                return paths, [None] * len(paths)
+
+        paths = check_urls_and_parts(paths, parts)
+        paths_and_parts = ensure_urls_and_parts(paths, parts, compress=True)
+
+        paths, parts = zip(*paths_and_parts)
+        assert len(paths) == len(parts)
+        if len(paths) == 1:
+            return paths[0], parts[0]
+        else:
+            return paths, parts
+
 
 class IndexedFileSource(FileSource):
     def mutate(self):
diff --git a/earthkit/data/sources/url.py b/earthkit/data/sources/url.py
index c71dbd8..efa97e1 100644
--- a/earthkit/data/sources/url.py
+++ b/earthkit/data/sources/url.py
@@ -16,6 +16,7 @@ from earthkit.data.core.caching import cache_file
 from earthkit.data.core.settings import SETTINGS
 from earthkit.data.core.statistics import record_statistics
 from earthkit.data.utils import progress_bar
+from earthkit.data.utils.parts import check_urls_and_parts, ensure_urls_and_parts
 
 from .file import FileSource
 
@@ -155,12 +156,6 @@ class UrlBase(FileSource):
 
         return headers
 
-    def _get_parts(self, url):
-        if isinstance(self.parts, dict):
-            return self.parts.get(url, None)
-        else:
-            return self.parts
-
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}({self.url})"
 
@@ -196,21 +191,21 @@ class Url(UrlBase):
         # TODO: re-enable this feature
         extension = None
 
-        self.prepare()
+        # put urls and parts into one list
+        self.urls_and_parts = self._urls_and_parts(self.url, self.parts)
 
         if not self.stream:
             self.update_if_out_of_date = update_if_out_of_date
 
             LOG.debug(
                 (
-                    f"url={self.url} parts={self.parts} auth={self.auth}) "
-                    f"http_headers={self.http_headers} parts_kwargs={self.parts_kwargs}"
+                    f"urls_and_parts={self.urls_and_parts} auth={self.auth}) "
+                    f"http_headers={self.http_headers}"
                     f" _kwargs={self._kwargs}"
                 )
             )
-
             self.downloader = Downloader(
-                self.url,
+                self.urls_and_parts,
                 chunk_size=self.chunk_size,
                 timeout=SETTINGS.get("url-download-timeout"),
                 verify=self.verify,
@@ -222,9 +217,10 @@ class Url(UrlBase):
                 resume_transfers=True,
                 override_target_file=False,
                 download_file_extension=".download",
-                **self.parts_kwargs,
             )
 
+            print(f"downloader={self.downloader}")
+
             if extension and extension[0] != ".":
                 extension = "." + extension
 
@@ -244,7 +240,7 @@ class Url(UrlBase):
 
             self.path = self.cache_file(
                 download,
-                dict(url=self.url, parts=self.parts),
+                dict(url=self.urls_and_parts),
                 extension=extension,
                 force=force,
             )
@@ -256,16 +252,9 @@ class Url(UrlBase):
 
     def mutate(self):
         if self.stream:
-            # create one stream source per url
-            from multiurl.downloader import _canonicalize
-
+            # # create one stream source per url
             s = []
-            _kwargs = dict(**self.parts_kwargs)
-            # if self.parts is not None:
-            #     _kwargs = {"parts": self.parts}
-            urls, _ = _canonicalize(self.url, **_kwargs)
-
-            for url, parts in urls:
+            for url, parts in self.urls_and_parts:
                 s.append(
                     SingleUrlStream(
                         url,
@@ -282,33 +271,43 @@ class Url(UrlBase):
 
             return _from_source(s, **self._kwargs)
         else:
+            # the underlying file source also try to use parts! So
+            # it has to be cleared to avoid using it again on the
+            # downloaded file!
+            self.parts = None
             return super().mutate()
 
-    def prepare(self):
-        # ensure no parts kwargs is used when the parts are defined together with the urls
-        self.parts_kwargs = {}
-        urls = self.url
-
-        if not isinstance(urls, (list, tuple)):
-            urls = [urls]
-
-        # a single url as [url, parts] is not allowed by multiurl
-        if (
-            len(urls) == 2
-            and isinstance(urls[0], str)
-            and (urls[1] is None or isinstance(urls[1], (list, tuple)))
-        ):
-            if self.parts is not None:
-                raise ValueError("Cannot specify parts both as arg and kwarg")
-            self.url, self.parts = urls
-            self.parts_kwargs = {"parts": self.parts}
-        # each url is a [url, parts]
-        elif isinstance(urls[0], (list, tuple)):
-            if self.parts is not None:
-                raise ValueError("Cannot specify parts both as arg and kwarg")
-        # each url is a str
-        else:
-            self.parts_kwargs = {"parts": self.parts}
+    @staticmethod
+    def _urls_and_parts(urls, parts):
+        """Preprocess urls and parts.
+
+        Parameters
+        ----------
+        urls: str or list/tuple
+            The url(s). When it is a sequence either each
+            item is a url (str), or a pair of a url and :ref:`parts <parts>`.
+        parts: part,list/tuple of parts or None.
+            The :ref:`parts <parts>`.
+
+        Returns
+        -------
+        list
+            Each item is a pair of url and part in a format expected by
+            ``multiurl``.  A part can be a
+            SimplePart, list/tuple of SimpleParts or None.
+
+        """
+        if parts is None:
+            if isinstance(urls, str):
+                return [[urls, None]]
+            elif isinstance(urls, (list, tuple)) and all(
+                isinstance(p, str) for p in urls
+            ):
+                return [(u, None) for u in urls]
+
+        urls = check_urls_and_parts(urls, parts)
+        urls_and_parts = ensure_urls_and_parts(urls, parts, compress=False)
+        return urls_and_parts
 
     def out_of_date(self, url, path, cache_data):
         if SETTINGS.get("check-out-of-date-urls") is False:
diff --git a/earthkit/data/utils/message.py b/earthkit/data/utils/message.py
index 6d840a3..aa3b776 100644
--- a/earthkit/data/utils/message.py
+++ b/earthkit/data/utils/message.py
@@ -74,25 +74,55 @@ def check_clone_kwargs(func):
 
 class CodesMessagePositionIndex:
     VERSION = 1
+    MAGIC = None
 
-    def __init__(self, path):
+    def __init__(self, path, parts=None):
         self.path = path
         self.offsets = None
         self.lengths = None
+        self.parts = parts
         self._cache_file = None
         self._load()
 
     def __len__(self):
         return len(self.offsets)
 
-    def _get_message_positions(self, path):
+    def _get_message_positions(self, path, parts):
+        fd = os.open(path, os.O_RDONLY)
+
+        try:
+            if parts is None:
+                yield from self._get_message_positions_part(fd, (0, -1))
+            else:
+                for part in parts:
+                    try:
+                        yield from self._get_message_positions_part(fd, part)
+                    except Exception:
+                        pass
+        except Exception:
+            pass
+        finally:
+            os.close(fd)
+
+    def _get_message_positions_part(self, path, part):
         raise NotImplementedError
 
+    @staticmethod
+    def _get_bytes(fd, count):
+        buf = os.read(fd, count)
+        if len(buf) != count:
+            raise Exception
+        return int.from_bytes(
+            buf,
+            byteorder="big",
+            signed=False,
+        )
+
     def _build(self):
         offsets = []
         lengths = []
 
-        for offset, length in self._get_message_positions(self.path):
+        for offset, length in self._get_message_positions(self.path, self.parts):
             offsets.append(offset)
             lengths.append(length)
 
diff --git a/earthkit/data/utils/parts.py b/earthkit/data/utils/parts.py
index 8b08829..52ea148 100644
--- a/earthkit/data/utils/parts.py
+++ b/earthkit/data/utils/parts.py
@@ -7,12 +7,14 @@
 # nor does it submit to any jurisdiction.
 
 import os
-from collections import defaultdict
+from collections import defaultdict, namedtuple
 
-# from earthkit.data.utils import download_and_cache
+SimplePart = namedtuple("SimplePart", ["offset", "length"])
 
 
 class Part:
+    """Represent a file part."""
+
     def __init__(self, path, offset, length):
         assert path is not None
         self.path = path
@@ -56,3 +58,94 @@ class Part:
 
     def __repr__(self):
         return f"Part[{self.path},{self.offset},{self.length}]"
+
+
+def check_urls_and_parts(urls, parts):
+    """Check if urls and parts are compatible
+
+    When any of the ``urls`` contain a part ``parts`` must be None.
+    """
+    if not isinstance(urls, (list, tuple)):
+        urls = [urls]
+
+    # a single url as [url, parts] is not allowed
+    if (
+        len(urls) == 2
+        and isinstance(urls[0], str)
+        and (urls[1] is None or isinstance(urls[1], (list, tuple)))
+    ):
+        if parts is not None:
+            raise ValueError("Cannot specify parts both as arg and kwarg")
+        urls = [urls]
+
+    if any(not isinstance(u, str) for u in urls):
+        if parts is not None:
+            raise ValueError("Cannot specify parts both as arg and kwarg")
+
+    return urls
+
+
+def _ensure_parts(parts):
+    if parts is None:
+        return None
+    if len(parts) == 2 and isinstance(parts[0], int) and isinstance(parts[1], int):
+        parts = [parts]
+    parts = [SimplePart(offset, length) for offset, length in parts]
+    if len(parts) == 0:
+        return None
+    return parts
+
+
+def ensure_urls_and_parts(urls, parts, compress=True):
+    if not isinstance(urls, (list, tuple)):
+        urls = [urls]
+
+    parts = _ensure_parts(parts)
+    result = []
+    for v in urls:
+        if isinstance(v, (list, tuple)):
+            u, p = v
+            result.append((u, _ensure_parts(p)))
+        else:
+            result.append((v, parts))
+
+    urls_and_parts = []
+    # Break into ascending order if needed
+    for url, parts in result:
+        if parts is None:
+            urls_and_parts.append((url, None))
+            continue
+
+        last = 0
+        newparts = []
+        for p in parts:
+            if p.offset < last:
+                if newparts:
+                    urls_and_parts.append((url, compress_parts(newparts)))
+                    newparts = []
+            newparts.append(p)
+            last = p.offset
+        urls_and_parts.append((url, compress_parts(newparts)))
+
+    return urls_and_parts
+
+
+def compress_parts(parts):
+    last = -1
+    result = []
+    # Compress and check
+    for offset, length in parts:
+        assert offset >= 0 and length > 0
+        if offset < last:
+            raise Exception(
+                f"Offsets and lengths must be in order, and not overlapping:"
+                f" offset={offset}, end of previous part={last}"
+            )
+        if offset == last:
+            # Compress
+            offset, prev_length = result.pop()
+            length += prev_length
+
+        result.append((offset, length))
+        last = offset + length
+    return tuple(SimplePart(offset, length) for offset, length in result)


Test patch:
diff --git a/tests/bufr/test_bufr_file_parts.py b/tests/bufr/test_bufr_file_parts.py
new file mode 100644
index 0000000..62dd22c
--- /dev/null
+++ b/tests/bufr/test_bufr_file_parts.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import pytest
+
+from earthkit.data import from_source
+from earthkit.data.testing import earthkit_examples_file
+
+
+@pytest.mark.parametrize(
+    "parts,expected_meta",
+    [
+        ((0, 1596), ["02836"]),  # message length
+        ([(0, 1596)], ["02836"]),  # message length
+        ([(0, 1601)], ["02836"]),  # message length + extra bytes
+        ([(2904, 1166)], ["01415"]),
+        ([(0, 1596)], ["02836"]),  # shorter than message
+        ([(2904, 2442)], ["01415", "01001"]),
+        ([(2904, 1166), (5346, 1216)], ["01415", "01152"]),
+        ([(2904, 1166), (5346, 121)], ["01415"]),  # second part shorter than message
+    ],
+)
+def test_bufr_single_file_parts(parts, expected_meta):
+    ds = from_source("file", earthkit_examples_file("temp_10.bufr"), parts=parts)
+
+    assert len(ds) == len(expected_meta)
+    if len(ds) > 0:
+        assert ds.metadata(("ident")) == expected_meta
+
+
+if __name__ == "__main__":
+    from earthkit.data.testing import main
+
+    main()
diff --git a/tests/grib/test_grib_file_parts.py b/tests/grib/test_grib_file_parts.py
new file mode 100644
index 0000000..1b5cc36
--- /dev/null
+++ b/tests/grib/test_grib_file_parts.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import pytest
+
+from earthkit.data import from_source
+from earthkit.data.testing import earthkit_examples_file
+
+
+@pytest.mark.parametrize(
+    "parts,expected_meta",
+    [
+        ((0, 150), [("t", 1000)]),  # message length
+        ([(0, 150)], [("t", 1000)]),  # message length
+        ([(0, 154)], [("t", 1000)]),  # message length + extra bytes
+        ([(0, 240)], [("t", 1000)]),  # message length + padding
+        (
+            [(0, 240)],
+            [("t", 1000)],
+        ),  # message length + padding + bytes from next message
+        ([(0, 15)], []),  # shorter than message
+        ([(240, 150)], [("u", 1000)]),
+        ([(240, 480)], [("u", 1000), ("v", 1000)]),
+        ([(240, 240), (720, 240)], [("u", 1000), ("t", 850)]),
+        ([(240, 240), (720, 243)], [("u", 1000), ("t", 850)]),
+        ([(240, 240), (720, 16)], [("u", 1000)]),  # second part shorter than message
+    ],
+)
+def test_grib_single_file_parts(parts, expected_meta):
+    ds = from_source("file", earthkit_examples_file("test6.grib"), parts=parts)
+
+    assert len(ds) == len(expected_meta)
+    if len(ds) > 0:
+        assert ds.metadata(("param", "level")) == expected_meta
+
+
+@pytest.mark.parametrize(
+    "parts1,parts2,expected_meta",
+    [
+        (
+            [(240, 150)],
+            None,
+            [("u", 1000), ("2t", 0), ("msl", 0)],
+        ),
+        (
+            (240, 150),
+            None,
+            [("u", 1000), ("2t", 0), ("msl", 0)],
+        ),
+        (
+            None,
+            [(0, 526)],
+            [
+                ("t", 1000),
+                ("u", 1000),
+                ("v", 1000),
+                ("t", 850),
+                ("u", 850),
+                ("v", 850),
+                ("2t", 0),
+            ],
+        ),
+        (
+            [(240, 150)],
+            [(0, 526)],
+            [("u", 1000), ("2t", 0)],
+        ),
+    ],
+)
+def test_grib_multi_file_parts(parts1, parts2, expected_meta):
+    ds = from_source(
+        "file",
+        [
+            [earthkit_examples_file("test6.grib"), parts1],
+            [earthkit_examples_file("test.grib"), parts2],
+        ],
+    )
+
+    assert len(ds) == len(expected_meta)
+    if len(ds) > 0:
+        assert ds.metadata(("param", "level")) == expected_meta
+
+
+if __name__ == "__main__":
+    from earthkit.data.testing import main
+
+    main()
diff --git a/tests/grib/test_grib_url.py b/tests/grib/test_grib_url.py
index 476a337..7ddc5c5 100644
--- a/tests/grib/test_grib_url.py
+++ b/tests/grib/test_grib_url.py
@@ -15,12 +15,23 @@ from earthkit.data import from_source
 from earthkit.data.testing import earthkit_remote_test_data_file
 
 
+# TODO: the disabled tests require a fix in multiurl
 @pytest.mark.parametrize(
     "parts,expected_meta",
     [
-        ([(0, 150)], [("t", 1000)]),
+        ([(0, 150)], [("t", 1000)]),  # message length
+        ([(0, 154)], [("t", 1000)]),  # message length + extra bytes
+        ([(0, 240)], [("t", 1000)]),  # message length + padding
+        (
+            [(0, 240)],
+            [("t", 1000)],
+        ),  # message length + padding + bytes from next message
+        # ([(0, 15)], []),  # shorter than message
         ([(240, 150)], [("u", 1000)]),
         ([(240, 480)], [("u", 1000), ("v", 1000)]),
+        ([(240, 240), (720, 240)], [("u", 1000), ("t", 850)]),
+        ([(240, 240), (720, 243)], [("u", 1000), ("t", 850)]),
+        # ([(240, 240), (720, 16)], [("u", 1000)]),  # second part shorter than message
     ],
 )
 def test_grib_single_url_parts(parts, expected_meta):
@@ -29,13 +40,8 @@ def test_grib_single_url_parts(parts, expected_meta):
     )
 
     assert len(ds) == len(expected_meta)
-
-    cnt = 0
-    for i, f in enumerate(ds):
-        assert f.metadata(("param", "level")) == expected_meta[i], i
-        cnt += 1
-
-    assert cnt == len(expected_meta)
+    if len(ds) > 0:
+        assert ds.metadata(("param", "level")) == expected_meta
 
 
 @pytest.mark.parametrize(
@@ -76,13 +82,7 @@ def test_grib_multi_url_parts(parts1, parts2, expected_meta):
     )
 
     assert len(ds) == len(expected_meta)
-
-    cnt = 0
-    for i, f in enumerate(ds):
-        assert f.metadata(("param", "level")) == expected_meta[i], i
-        cnt += 1
-
-    assert cnt == len(expected_meta)
+    assert ds.metadata(("param", "level")) == expected_meta
 
 
 if __name__ == "__main__":
diff --git a/tests/utils/test_parts.py b/tests/utils/test_parts.py
new file mode 100644
index 0000000..337454d
--- /dev/null
+++ b/tests/utils/test_parts.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+
+import pytest
+
+from earthkit.data.sources.file import FileSource
+from earthkit.data.sources.url import Url
+from earthkit.data.utils.parts import SimplePart
+
+
+@pytest.mark.parametrize(
+    "paths,parts,expected_paths,expected_parts",
+    [
+        ("a.grib", None, "a.grib", None),
+        (["a.grib", None], None, "a.grib", None),
+        (["a.grib", [(0, 150)]], None, "a.grib", (SimplePart(0, 150),)),
+        (["a.grib", (0, 150)], None, "a.grib", (SimplePart(0, 150),)),
+        ("a.grib", [(0, 150)], "a.grib", (SimplePart(0, 150),)),
+        ("a.grib", [(0, 150), (150, 100)], "a.grib", (SimplePart(0, 250),)),
+        (
+            "a.grib",
+            [(0, 150), (160, 100)],
+            "a.grib",
+            (SimplePart(0, 150), SimplePart(160, 100)),
+        ),
+        (
+            "a.grib",
+            [(150, 100), (0, 150)],
+            ("a.grib", "a.grib"),
+            ((SimplePart(150, 100),), (SimplePart(0, 150),)),
+        ),
+        (["a.grib", "b.grib"], None, ["a.grib", "b.grib"], [None, None]),
+        (
+            ["a.grib", "b.grib"],
+            (0, 150),
+            ("a.grib", "b.grib"),
+            ((SimplePart(0, 150),), (SimplePart(0, 150),)),
+        ),
+        (
+            [["a.grib", [(0, 150), (200, 20)]], "b.grib"],
+            None,
+            ("a.grib", "b.grib"),
+            ((SimplePart(0, 150), SimplePart(200, 20)), None),
+        ),
+        (
+            [["a.grib", [(0, 150), (200, 20)]], ["b.grib", (0, 100)]],
+            None,
+            ("a.grib", "b.grib"),
+            ((SimplePart(0, 150), SimplePart(200, 20)), ((SimplePart(0, 100),))),
+        ),
+        (
+            [["a.grib", [(0, 150), (200, 20)]], ["b.grib", [(0, 100)]]],
+            None,
+            ("a.grib", "b.grib"),
+            ((SimplePart(0, 150), SimplePart(200, 20)), ((SimplePart(0, 100),))),
+        ),
+        (
+            [["a.grib", [(0, 150), (200, 20)]], ["b.grib", [(0, 100), (120, 50)]]],
+            None,
+            ("a.grib", "b.grib"),
+            (
+                (SimplePart(0, 150), SimplePart(200, 20)),
+                ((SimplePart(0, 100), SimplePart(120, 50))),
+            ),
+        ),
+        (
+            ["a.grib", "b.grib"],
+            (0, 150),
+            ("a.grib", "b.grib"),
+            ((SimplePart(0, 150),), (SimplePart(0, 150),)),
+        ),
+    ],
+)
+def test_prepare_file_parts(paths, parts, expected_paths, expected_parts):
+    res_paths, res_parts = FileSource._paths_and_parts(paths, parts)
+    assert res_paths == expected_paths
+    assert res_parts == expected_parts
+
+
+@pytest.mark.parametrize(
+    "urls,parts,expected_values",
+    [
+        ("a.grib", None, [["a.grib", None]]),
+        (["a.grib", None], None, [("a.grib", None)]),
+        (["a.grib", [(0, 150)]], None, [("a.grib", (SimplePart(0, 150),))]),
+        (["a.grib", (0, 150)], None, [("a.grib", (SimplePart(0, 150),))]),
+        ("a.grib", [(0, 150)], [("a.grib", (SimplePart(0, 150),))]),
+        ("a.grib", [(0, 150), (150, 100)], [("a.grib", (SimplePart(0, 250),))]),
+        (
+            "a.grib",
+            [(0, 150), (160, 100)],
+            [("a.grib", (SimplePart(0, 150), SimplePart(160, 100)))],
+        ),
+        (
+            "a.grib",
+            [(150, 100), (0, 150)],
+            [
+                ("a.grib", ((SimplePart(150, 100),))),
+                ("a.grib", ((SimplePart(0, 150),))),
+            ],
+        ),
+        (["a.grib", "b.grib"], None, [("a.grib", None), ("b.grib", None)]),
+        (
+            ["a.grib", "b.grib"],
+            (0, 150),
+            [("a.grib", (SimplePart(0, 150),)), ("b.grib", (SimplePart(0, 150),))],
+        ),
+        (
+            [["a.grib", [(0, 150), (200, 20)]], "b.grib"],
+            None,
+            [("a.grib", (SimplePart(0, 150), SimplePart(200, 20))), ("b.grib", None)],
+        ),
+        (
+            [["a.grib", [(0, 150), (200, 20)]], ["b.grib", (0, 100)]],
+            None,
+            [
+                ("a.grib", (SimplePart(0, 150), SimplePart(200, 20))),
+                ("b.grib", (SimplePart(0, 100),)),
+            ],
+        ),
+        (
+            [["a.grib", [(0, 150), (200, 20)]], ["b.grib", [(0, 100)]]],
+            None,
+            [
+                ("a.grib", (SimplePart(0, 150), SimplePart(200, 20))),
+                ("b.grib", (SimplePart(0, 100),)),
+            ],
+        ),
+        (
+            [["a.grib", [(0, 150), (200, 20)]], ["b.grib", [(0, 100), (120, 50)]]],
+            None,
+            [
+                ("a.grib", (SimplePart(0, 150), SimplePart(200, 20))),
+                ("b.grib", (SimplePart(0, 100), SimplePart(120, 50))),
+            ],
+        ),
+        (
+            ["a.grib", "b.grib"],
+            (0, 150),
+            [
+                ("a.grib", (SimplePart(0, 150),)),
+                ("b.grib", (SimplePart(0, 150),)),
+            ],
+        ),
+    ],
+)
+def test_prepare_url_parts(urls, parts, expected_values):
+    res = Url._urls_and_parts(urls, parts)
+    assert res == expected_values
+
+
+if __name__ == "__main__":
+    from earthkit.data.testing import main
+
+    main(__file__)