Repository: UCL-Chimera/medical-waveforms (language: python)

Issue:
Support `find_troughs()` for long signals with limited RAM
Currently runs out of memory in this context, but could process in chunks
Poetry install fails for python 3.6
This makes the tests fail. It seems that poetry has dropped python 3.6 support: https://github.com/python-poetry/poetry/issues/4997


Discussion / hints:
@JMathiszig-Lee suggests using generators to yield each chunk
Main memory-hungry step in `find_troughs()` is allocation of the LSM matrix which is (x, x // 2) where x is the number of timesteps. Could use this calculation to calculate chunk size to fit in a specified amount of RAM


Fix patch:
diff --git a/examples/get_cycles_benchmarking.ipynb b/examples/get_cycles_benchmarking.ipynb
index 1a2215e..319f8ac 100644
--- a/examples/get_cycles_benchmarking.ipynb
+++ b/examples/get_cycles_benchmarking.ipynb
@@ -11,7 +11,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -37,7 +37,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -45,11 +45,89 @@
     "    systolic_pressure=120,\n",
     "    diastolic_pressure=80,\n",
     "    heart_rate=70,\n",
-    "    n_beats_target=20 * 60 / (60 / 70),\n",
+    "    n_beats_target=100,\n",
     "    hertz=100\n",
     ")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>time</th>\n",
+       "      <th>pressure</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.00</td>\n",
+       "      <td>80.036900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0.01</td>\n",
+       "      <td>80.491320</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0.02</td>\n",
+       "      <td>82.081806</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0.03</td>\n",
+       "      <td>85.217117</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0.04</td>\n",
+       "      <td>89.627306</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   time   pressure\n",
+       "0  0.00  80.036900\n",
+       "1  0.01  80.491320\n",
+       "2  0.02  82.081806\n",
+       "3  0.03  85.217117\n",
+       "4  0.04  89.627306"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data.head()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -74,7 +152,7 @@
     {
      "data": {
       "text/plain": [
-       "240002"
+       "17144"
       ]
      },
      "execution_count": 5,
@@ -101,19 +179,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "wf = find_troughs(wf, name='pressure')"
+    "wf = find_troughs(wf, name='pressure', scale=50)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "34.1 ms ± 437 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
+      "1.05 ms ± 36.6 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)\n"
      ]
     }
    ],
@@ -132,7 +210,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.10.2 ('sidewinder-axJzVSFK-py3.10')",
+   "display_name": "Python 3.9.9 ('sidewinder-lxfscp8q-py3.9')",
    "language": "python",
    "name": "python3"
   },
@@ -146,12 +224,12 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.2"
+   "version": "3.9.9"
   },
   "orig_nbformat": 4,
   "vscode": {
    "interpreter": {
-    "hash": "52b33a274511e606d62a0c6c9d6ddb3dbbb94c75bbbe067dfc3130d53f815d2e"
+    "hash": "f2c1554fc2f673ff3983ee6916670176b140c542ca5d14124814c0e65b9d8c04"
    }
   }
  },
diff --git a/examples/tutorial.ipynb b/examples/tutorial.ipynb
index 52d13a2..0bca877 100644
--- a/examples/tutorial.ipynb
+++ b/examples/tutorial.ipynb
@@ -185,7 +185,9 @@
    "source": [
     "### Find troughs in a waveform\n",
     "\n",
-    "In this case, we use the troughs to split our arterial blood pressure data into separate heartbeats."
+    "In this case, we use the troughs to split our arterial blood pressure data into separate heartbeats.\n",
+    "\n",
+    "NB. Specifying the `scale` argument of `find_troughs` can save lots of memory if your data contain many cycles. Here I set it to (very approximately) half of the cycle length."
    ]
   },
   {
@@ -194,7 +196,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "wf = find_troughs(wf, name='pressure')"
+    "wf = find_troughs(wf, name='pressure', scale=100)"
    ]
   },
   {
@@ -327,7 +329,7 @@
    "outputs": [],
    "source": [
     "wf_with_flush = Waveforms(data_with_flush)\n",
-    "wf_with_flush = find_troughs(wf_with_flush, \"pressure\")"
+    "wf_with_flush = find_troughs(wf_with_flush, \"pressure\", scale=50)"
    ]
   },
   {
diff --git a/sidewinder/features/waveform.py b/sidewinder/features/waveform.py
index 6046cfb..dc56f26 100644
--- a/sidewinder/features/waveform.py
+++ b/sidewinder/features/waveform.py
@@ -1,22 +1,39 @@
+from typing import Optional
+from warnings import warn
+
 from pyampd import ampd
 
 from ..waveforms import Waveforms
 
 
-def find_troughs(waveforms: Waveforms, name: str) -> Waveforms:
+def find_troughs(
+    waveforms: Waveforms, name: str, scale: Optional[int] = None
+) -> Waveforms:
     """Finds indices of troughs in a waveform.
 
     Args:
         waveforms: `sidewinder.waveforms.Waveforms` instance holding your data
         name: Name of column in `waveforms` to find troughs in
+        scale : The maximum scale window size is (2 * scale + 1) during trough
+            finding. Higher values require more memory. If None, sets `scale`
+            to number_of_timesteps // 2 but often much lower values are
+            adequate if your data contain many cycle. E.g.
+            https://link.springer.com/chapter/10.1007/978-3-319-65798-1_39 uses
+            `scale` values around one quarter of the cycle length for#
+            intracranial pressure waveforms.
 
     Returns:
         `waveforms` with trough indices added to
             `waveforms.features.waveform[`name`]['troughs']`
     """
     waveforms.waveforms[name] *= -1  # invert signal, so troughs become peaks
-    waveforms.features.waveform[name]["troughs"] = ampd.find_peaks(
-        waveforms.waveforms[name]
-    )
+
+    try:
+        waveforms.features.waveform[name]["troughs"] = ampd.find_peaks(
+            x=waveforms.waveforms[name], scale=scale
+        )
+    except MemoryError:
+        warn("Ran out of memory. Try setting `scale` to a lower value.")
+
     waveforms.waveforms[name] *= -1  # un-invert signal
     return waveforms


Test patch:
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index effa961..1f8e8e7 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -9,7 +9,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3