{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from pathlib import Path\n", "\n", "from ase import units, Atoms\n", "from ase.build import molecule\n", "from ase.io import read, write\n", "from dask.distributed import Client\n", "from dask_jobqueue import SLURMCluster\n", "from prefect import flow\n", "from prefect_dask import DaskTaskRunner\n", "from pymatgen.core import Molecule\n", "from pymatgen.io.packmol import PackmolBoxGen\n", "\n", "from mlip_arena.models.utils import REGISTRY, MLIPEnum\n", "from mlip_arena.tasks.run import md as MD" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "h2 = molecule(\"H2\")\n", "o2 = molecule(\"O2\")\n", "h2o = molecule(\"H2O\")\n", "\n", "write(\"h2.xyz\", h2)\n", "write(\"o2.xyz\", o2)\n", "write(\"h2o.xyz\", h2o)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "h2 = Molecule.from_file(\"h2.xyz\")\n", "o2 = Molecule.from_file(\"o2.xyz\")\n", "h2o = Molecule.from_file(\"h2o.xyz\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "molecules = []\n", "\n", "for m, number in zip([h2, o2], [128, 64]):\n", " molecules.append(\n", " {\n", " \"name\": m.composition.to_pretty_string(),\n", " \"number\": number,\n", " \"coords\": m,\n", " }\n", " )" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "tolerance = 2.0\n", "input_gen = PackmolBoxGen(\n", " tolerance=tolerance,\n", " seed=1,\n", ")\n", "margin = 0.5 * tolerance\n", "\n", "a = 30\n", "\n", "packmol_set = input_gen.get_input_set(\n", " molecules=molecules,\n", " box=[margin, margin, margin, a - margin, a - margin, a - margin],\n", ")\n", "packmol_set.write_input(\".\")\n", "packmol_set.run(\".\")\n", "\n", "atoms = read(\"packmol_out.xyz\")\n", "atoms.cell = [a, a, a]\n", "atoms.pbc = True\n", "\n", "print(atoms)\n", "\n", "write(f'{atoms.get_chemical_formula()}.extxyz', atoms)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Atoms(symbols='H256O128', pbc=True, cell=[30.0, 30.0, 30.0])\n" ] } ], "source": [ "atoms = read(\"H256O128.extxyz\")\n", "print(atoms)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "#!/bin/bash\n", "\n", "#SBATCH -A matgen\n", "#SBATCH --mem=0\n", "#SBATCH -t 02:00:00\n", "#SBATCH -J combustion-water\n", "#SBATCH -q regular\n", "#SBATCH -N 1\n", "#SBATCH -C gpu\n", "#SBATCH -G 4\n", "#SBATCH --exclusive\n", "source ~/.bashrc\n", "module load python\n", "source activate /pscratch/sd/c/cyrusyc/.conda/mlip-arena\n", "/pscratch/sd/c/cyrusyc/.conda/mlip-arena/bin/python -m distributed.cli.dask_worker tcp://128.55.64.18:43951 --name dummy-name --nthreads 1 --memory-limit 59.60GiB --nanny --death-timeout 86400\n", "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2024-09-27 03:54:04,812 - distributed.scheduler - ERROR - Task md-2-9b52cfa3e22647de99659385eca3d3f2-1 marked as failed because 4 workers died while trying to run it\n", "2024-09-27 03:54:04,820 - distributed.scheduler - ERROR - Task md-5-2a33088343d64b66b612e4145ffaf870-1 marked as failed because 4 workers died while trying to run it\n", "2024-09-27 03:54:04,824 - distributed.scheduler - ERROR - Task md-0-017eef8ab8744d7c958badc5d53fdab4-1 marked as failed because 4 workers died while trying to run it\n", "2024-09-27 05:55:56,468 - distributed.scheduler - ERROR - Task md-8-bcd5e0bb82854952b2c5c430d4c2e7e8-1 marked as failed because 4 workers died while trying to run it\n", "2024-09-27 05:55:56,906 - distributed.scheduler - ERROR - Task md-6-a575c0cc3b1e4fc984560ff101dbfc6c-1 marked as failed because 4 workers died while trying to run it\n", "2024-09-27 05:55:56,907 - distributed.scheduler - ERROR - Task md-9-f98b1f7c61c749c8b67cd669517a49bd-1 marked as failed because 4 workers died while trying to run it\n", "2024-09-27 05:55:56,910 - distributed.scheduler - ERROR - Task md-4-0e1acf2be8994ca7ae198d52d5e04d61-1 marked as failed because 4 workers died while trying to run it\n", "2024-09-27 05:57:02,776 - distributed.scheduler - ERROR - Task md-7-40b7b4ede41745e29d5c5fd0cc4d9e42-1 marked as failed because 4 workers died while trying to run it\n" ] } ], "source": [ "nodes_per_alloc = 1\n", "gpus_per_alloc = 4\n", "ntasks = 1\n", "\n", "cluster_kwargs = dict(\n", " cores=1,\n", " memory=\"64 GB\",\n", " shebang=\"#!/bin/bash\",\n", " account=\"matgen\",\n", " walltime=\"02:00:00\",\n", " job_mem=\"0\",\n", " job_script_prologue=[\n", " \"source ~/.bashrc\",\n", " \"module load python\",\n", " \"source activate /pscratch/sd/c/cyrusyc/.conda/mlip-arena\",\n", " ],\n", " job_directives_skip=[\"-n\", \"--cpus-per-task\", \"-J\"],\n", " job_extra_directives=[\n", " \"-J combustion-water\",\n", " \"-q regular\",\n", " f\"-N {nodes_per_alloc}\",\n", " \"-C gpu\",\n", " f\"-G {gpus_per_alloc}\",\n", " f\"--exclusive\",\n", " # \"--time-min=00:30:00\",\n", " # \"--comment=1-00:00:00\",\n", " # \"--signal=B:USR1@60\",\n", " # \"--requeue\",\n", " # \"--open-mode=append\"\n", " ],\n", " death_timeout=86400\n", ")\n", "\n", "\n", "cluster = SLURMCluster(**cluster_kwargs)\n", "print(cluster.job_script())\n", "cluster.adapt(minimum_jobs=10, maximum_jobs=11)\n", "client = Client(cluster)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "@flow(task_runner=DaskTaskRunner(address=client.scheduler.address), log_prints=True)\n", "def combustion(atoms: Atoms):\n", " futures = []\n", "\n", " for model in MLIPEnum:\n", " future = MD.submit(\n", " atoms=atoms,\n", " calculator_name=model,\n", " calculator_kwargs=None,\n", " ensemble=\"nvt\",\n", " dynamics=\"nose-hoover\",\n", " time_step=None,\n", " ase_md_kwargs=dict(ttime=25 * units.fs, pfactor=None),\n", " total_time=1000_000,\n", " temperature=[300, 3000, 3000, 300],\n", " pressure=None,\n", " mb_velocity_seed=0,\n", " traj_file=Path(REGISTRY[model.name][\"family\"])\n", " / f\"{model.name}_{atoms.get_chemical_formula()}.traj\",\n", " traj_interval=1000,\n", " restart=True,\n", " )\n", "\n", " futures.append(future)\n", " \n", " return [future.result() for future in futures]" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
23:19:58.587 | INFO    | prefect.engine - Created flow run 'tangerine-mussel' for flow 'combustion'\n",
       "
\n" ], "text/plain": [ "23:19:58.587 | \u001b[36mINFO\u001b[0m | prefect.engine - Created flow run\u001b[35m 'tangerine-mussel'\u001b[0m for flow\u001b[1;35m 'combustion'\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:58.592 | INFO    | Flow run 'tangerine-mussel' - View at https://app.prefect.cloud/account/f7d40474-9362-4bfa-8950-ee6a43ec00f3/workspace/d4bb0913-5f5e-49f7-bfc5-06509088baeb/flow-runs/flow-run/ea339a90-929f-491d-bf23-e257c925ad6b\n",
       "
\n" ], "text/plain": [ "23:19:58.592 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - View at \u001b[94mhttps://app.prefect.cloud/account/f7d40474-9362-4bfa-8950-ee6a43ec00f3/workspace/d4bb0913-5f5e-49f7-bfc5-06509088baeb/flow-runs/flow-run/ea339a90-929f-491d-bf23-e257c925ad6b\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:58.593 | INFO    | prefect.task_runner.dask - Connecting to an existing Dask cluster at tcp://128.55.64.18:43951\n",
       "
\n" ], "text/plain": [ "23:19:58.593 | \u001b[36mINFO\u001b[0m | prefect.task_runner.dask - Connecting to an existing Dask cluster at tcp://128.55.64.18:43951\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:58.599 | INFO    | prefect.task_runner.dask - The Dask dashboard is available at http://128.55.64.18:8787/status\n",
       "
\n" ], "text/plain": [ "23:19:58.599 | \u001b[36mINFO\u001b[0m | prefect.task_runner.dask - The Dask dashboard is available at \u001b[94mhttp://128.55.64.18:8787/status\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:59.454 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-2' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:19:59.454 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-2' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:59.484 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-6' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:19:59.484 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-6' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:59.488 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-3' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:19:59.488 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-3' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:59.491 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-5' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:19:59.491 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-5' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:59.494 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-9' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:19:59.494 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-9' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:59.497 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-1' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:19:59.497 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-1' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:59.646 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-0' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:19:59.646 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-0' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:19:59.938 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-7' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:19:59.938 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-7' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.111 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-5' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.111 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-5' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.119 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-8' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:20:00.119 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-8' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.125 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-8' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.125 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-8' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.131 | INFO    | Flow run 'tangerine-mussel' - Created task run 'md-4' for task 'md'\n",
       "
\n" ], "text/plain": [ "23:20:00.131 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Created task run 'md-4' for task 'md'\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.137 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-4' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.137 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-4' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.154 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-6' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.154 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-6' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.205 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-0' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.205 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-0' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.212 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-1' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.212 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-1' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.283 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-2' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.283 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-2' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.291 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-9' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.291 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-9' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.400 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-3' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.400 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-3' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
23:20:00.554 | INFO    | Flow run 'tangerine-mussel' - Submitted task run 'md-7' for execution.\n",
       "
\n" ], "text/plain": [ "23:20:00.554 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Submitted task run 'md-7' for execution.\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
03:54:04.823 | ERROR   | Flow run 'tangerine-mussel' - Encountered exception during execution:\n",
       "Traceback (most recent call last):\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py\", line 867, in orchestrate_flow_run\n",
       "    result = await flow_call.aresult()\n",
       "             ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 327, in aresult\n",
       "    return await asyncio.wrap_future(self.future)\n",
       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 352, in _run_sync\n",
       "    result = self.fn(*self.args, **self.kwargs)\n",
       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/tmp/ipykernel_2266116/1920202683.py\", line 26, in combustion\n",
       "    return [future.result() for future in futures]\n",
       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/tmp/ipykernel_2266116/1920202683.py\", line 26, in <listcomp>\n",
       "    return [future.result() for future in futures]\n",
       "            ^^^^^^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py\", line 228, in result\n",
       "    return from_sync.call_soon_in_loop_thread(result).result()\n",
       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 318, in result\n",
       "    return self.future.result(timeout=timeout)\n",
       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 188, in result\n",
       "    return self.__get_result()\n",
       "           ^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/concurrent/futures/_base.py\", line 401, in __get_result\n",
       "    raise self._exception\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 389, in _run_async\n",
       "    result = await coro\n",
       "             ^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py\", line 237, in _result\n",
       "    return await final_state.result(raise_on_failure=raise_on_failure, fetch=True)\n",
       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/states.py\", line 91, in _get_state_result\n",
       "    raise await get_state_exception(state)\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect_dask/task_runners.py\", line 311, in wait\n",
       "    return await future.result(timeout=timeout)\n",
       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
       "  File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/distributed/client.py\", line 336, in _result\n",
       "    raise exc.with_traceback(tb)\n",
       "distributed.scheduler.KilledWorker: Attempted to run task 'md-2-9b52cfa3e22647de99659385eca3d3f2-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.\n",
       "
\n" ], "text/plain": [ "03:54:04.823 | \u001b[38;5;160mERROR\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Encountered exception during execution:\n", "Traceback (most recent call last):\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py\", line 867, in orchestrate_flow_run\n", " result = await flow_call.aresult()\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 327, in aresult\n", " return await asyncio.wrap_future(self.future)\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 352, in _run_sync\n", " result = self.fn(*self.args, **self.kwargs)\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/tmp/ipykernel_2266116/1920202683.py\", line 26, in combustion\n", " return [future.result() for future in futures]\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/tmp/ipykernel_2266116/1920202683.py\", line 26, in \n", " return [future.result() for future in futures]\n", " ^^^^^^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py\", line 228, in result\n", " return from_sync.call_soon_in_loop_thread(result).result()\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 318, in result\n", " return self.future.result(timeout=timeout)\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 188, in result\n", " return self.__get_result()\n", " ^^^^^^^^^^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/concurrent/futures/_base.py\", line 401, in __get_result\n", " raise self._exception\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 389, in _run_async\n", " result = await coro\n", " ^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py\", line 237, in _result\n", " return await final_state.result(raise_on_failure=raise_on_failure, fetch=True)\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/states.py\", line 91, in _get_state_result\n", " raise await get_state_exception(state)\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect_dask/task_runners.py\", line 311, in wait\n", " return await future.result(timeout=timeout)\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/distributed/client.py\", line 336, in _result\n", " raise exc.with_traceback(tb)\n", "distributed.scheduler.KilledWorker: Attempted to run task 'md-2-9b52cfa3e22647de99659385eca3d3f2-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
05:57:03.100 | INFO    | Task run 'md-2' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-2-9b52cfa3e22647de99659385eca3d3f2-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.\n",
       "
\n" ], "text/plain": [ "05:57:03.100 | \u001b[36mINFO\u001b[0m | Task run 'md-2' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-2-9b52cfa3e22647de99659385eca3d3f2-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
05:57:03.472 | INFO    | Task run 'md-5' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-5-2a33088343d64b66b612e4145ffaf870-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.\n",
       "
\n" ], "text/plain": [ "05:57:03.472 | \u001b[36mINFO\u001b[0m | Task run 'md-5' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-5-2a33088343d64b66b612e4145ffaf870-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
05:57:03.991 | INFO    | Task run 'md-6' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-6-a575c0cc3b1e4fc984560ff101dbfc6c-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.100:45983. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.\n",
       "
\n" ], "text/plain": [ "05:57:03.991 | \u001b[36mINFO\u001b[0m | Task run 'md-6' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-6-a575c0cc3b1e4fc984560ff101dbfc6c-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.100:45983. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
05:57:04.333 | INFO    | Task run 'md-7' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-7-40b7b4ede41745e29d5c5fd0cc4d9e42-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.68.6:39739. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.\n",
       "
\n" ], "text/plain": [ "05:57:04.333 | \u001b[36mINFO\u001b[0m | Task run 'md-7' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-7-40b7b4ede41745e29d5c5fd0cc4d9e42-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.68.6:39739. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
05:57:04.976 | INFO    | Task run 'md-8' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-8-bcd5e0bb82854952b2c5c430d4c2e7e8-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.238:37065. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.\n",
       "
\n" ], "text/plain": [ "05:57:04.976 | \u001b[36mINFO\u001b[0m | Task run 'md-8' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-8-bcd5e0bb82854952b2c5c430d4c2e7e8-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.238:37065. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
05:57:05.435 | INFO    | Task run 'md-9' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-9-f98b1f7c61c749c8b67cd669517a49bd-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.65.221:34843. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.\n",
       "
\n" ], "text/plain": [ "05:57:05.435 | \u001b[36mINFO\u001b[0m | Task run 'md-9' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-9-f98b1f7c61c749c8b67cd669517a49bd-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.65.221:34843. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
05:57:05.840 | ERROR   | Flow run 'tangerine-mussel' - Finished in state Failed(\"Flow run encountered an exception. KilledWorker: Attempted to run task 'md-2-9b52cfa3e22647de99659385eca3d3f2-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.\")\n",
       "
\n" ], "text/plain": [ "05:57:05.840 | \u001b[38;5;160mERROR\u001b[0m | Flow run\u001b[35m 'tangerine-mussel'\u001b[0m - Finished in state \u001b[38;5;160mFailed\u001b[0m(\"Flow run encountered an exception. KilledWorker: Attempted to run task 'md-2-9b52cfa3e22647de99659385eca3d3f2-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\")\n" ] }, "metadata": {}, "output_type": "display_data" }, { "ename": "KilledWorker", "evalue": "Attempted to run task 'md-2-9b52cfa3e22647de99659385eca3d3f2-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKilledWorker\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mcombustion\u001b[49m\u001b[43m(\u001b[49m\u001b[43matoms\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/flows.py:1224\u001b[0m, in \u001b[0;36mFlow.__call__\u001b[0;34m(self, return_state, wait_for, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1219\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m task_viz_tracker:\n\u001b[1;32m 1220\u001b[0m \u001b[38;5;66;03m# this is a subflow, for now return a single task and do not go further\u001b[39;00m\n\u001b[1;32m 1221\u001b[0m \u001b[38;5;66;03m# we can add support for exploring subflows for tasks in the future.\u001b[39;00m\n\u001b[1;32m 1222\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m track_viz_task(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39misasync, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname, parameters)\n\u001b[0;32m-> 1224\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43menter_flow_run_engine_from_flow_call\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1225\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1226\u001b[0m \u001b[43m \u001b[49m\u001b[43mparameters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1227\u001b[0m \u001b[43m \u001b[49m\u001b[43mwait_for\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mwait_for\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1228\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1229\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py:297\u001b[0m, in \u001b[0;36menter_flow_run_engine_from_flow_call\u001b[0;34m(flow, parameters, wait_for, return_type)\u001b[0m\n\u001b[1;32m 290\u001b[0m retval \u001b[38;5;241m=\u001b[39m from_async\u001b[38;5;241m.\u001b[39mwait_for_call_in_loop_thread(\n\u001b[1;32m 291\u001b[0m begin_run,\n\u001b[1;32m 292\u001b[0m done_callbacks\u001b[38;5;241m=\u001b[39mdone_callbacks,\n\u001b[1;32m 293\u001b[0m contexts\u001b[38;5;241m=\u001b[39mcontexts,\n\u001b[1;32m 294\u001b[0m )\n\u001b[1;32m 296\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 297\u001b[0m retval \u001b[38;5;241m=\u001b[39m \u001b[43mfrom_sync\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_for_call_in_loop_thread\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 298\u001b[0m \u001b[43m \u001b[49m\u001b[43mbegin_run\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 299\u001b[0m \u001b[43m \u001b[49m\u001b[43mdone_callbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdone_callbacks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 300\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontexts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontexts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 301\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m retval\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/api.py:243\u001b[0m, in \u001b[0;36mfrom_sync.wait_for_call_in_loop_thread\u001b[0;34m(_from_sync__call, timeout, done_callbacks, contexts)\u001b[0m\n\u001b[1;32m 241\u001b[0m stack\u001b[38;5;241m.\u001b[39menter_context(context)\n\u001b[1;32m 242\u001b[0m waiter\u001b[38;5;241m.\u001b[39mwait()\n\u001b[0;32m--> 243\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:318\u001b[0m, in \u001b[0;36mCall.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mresult\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout: Optional[\u001b[38;5;28mfloat\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m 313\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;124;03m Wait for the result of the call.\u001b[39;00m\n\u001b[1;32m 315\u001b[0m \n\u001b[1;32m 316\u001b[0m \u001b[38;5;124;03m Not safe for use from asynchronous contexts.\u001b[39;00m\n\u001b[1;32m 317\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 318\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:179\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[1;32m 178\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m--> 179\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 181\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_condition\u001b[38;5;241m.\u001b[39mwait(timeout)\n\u001b[1;32m 183\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n\u001b[1;32m 184\u001b[0m \u001b[38;5;66;03m# Raise Prefect cancelled error instead of\u001b[39;00m\n\u001b[1;32m 185\u001b[0m \u001b[38;5;66;03m# `concurrent.futures._base.CancelledError`\u001b[39;00m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/concurrent/futures/_base.py:401\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n\u001b[1;32m 400\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 401\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 403\u001b[0m \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:389\u001b[0m, in \u001b[0;36mCall._run_async\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 387\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39menforce_async_deadline() \u001b[38;5;28;01mas\u001b[39;00m cancel_scope:\n\u001b[1;32m 388\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 389\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 390\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 391\u001b[0m \u001b[38;5;66;03m# Forget this call's arguments in order to free up any memory\u001b[39;00m\n\u001b[1;32m 392\u001b[0m \u001b[38;5;66;03m# that may be referenced by them; after a call has happened,\u001b[39;00m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;66;03m# there's no need to keep a reference to them\u001b[39;00m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/client/utilities.py:78\u001b[0m, in \u001b[0;36minject_client..with_injected_client\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m context \u001b[38;5;28;01mas\u001b[39;00m new_client:\n\u001b[1;32m 77\u001b[0m kwargs\u001b[38;5;241m.\u001b[39msetdefault(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclient\u001b[39m\u001b[38;5;124m\"\u001b[39m, new_client \u001b[38;5;129;01mor\u001b[39;00m client)\n\u001b[0;32m---> 78\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m fn(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py:400\u001b[0m, in \u001b[0;36mcreate_then_begin_flow_run\u001b[0;34m(flow, parameters, wait_for, return_type, client, user_thread)\u001b[0m\n\u001b[1;32m 398\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m return_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresult\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 400\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m state\u001b[38;5;241m.\u001b[39mresult(fetch\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 401\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid return type for flow engine \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreturn_type\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/states.py:91\u001b[0m, in \u001b[0;36m_get_state_result\u001b[0;34m(state, raise_on_failure)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m UnfinishedRun(\n\u001b[1;32m 85\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRun is in \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstate\u001b[38;5;241m.\u001b[39mtype\u001b[38;5;241m.\u001b[39mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m state, its result is not available.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 86\u001b[0m )\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m raise_on_failure \u001b[38;5;129;01mand\u001b[39;00m (\n\u001b[1;32m 89\u001b[0m state\u001b[38;5;241m.\u001b[39mis_crashed() \u001b[38;5;129;01mor\u001b[39;00m state\u001b[38;5;241m.\u001b[39mis_failed() \u001b[38;5;129;01mor\u001b[39;00m state\u001b[38;5;241m.\u001b[39mis_cancelled()\n\u001b[1;32m 90\u001b[0m ):\n\u001b[0;32m---> 91\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m get_state_exception(state)\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(state\u001b[38;5;241m.\u001b[39mdata, DataDocument):\n\u001b[1;32m 94\u001b[0m result \u001b[38;5;241m=\u001b[39m result_from_state_with_data_document(\n\u001b[1;32m 95\u001b[0m state, raise_on_failure\u001b[38;5;241m=\u001b[39mraise_on_failure\n\u001b[1;32m 96\u001b[0m )\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py:867\u001b[0m, in \u001b[0;36morchestrate_flow_run\u001b[0;34m(flow, flow_run, parameters, wait_for, interruptible, client, partial_flow_run_context, user_thread)\u001b[0m\n\u001b[1;32m 862\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 863\u001b[0m from_async\u001b[38;5;241m.\u001b[39mcall_soon_in_new_thread(\n\u001b[1;32m 864\u001b[0m flow_call, timeout\u001b[38;5;241m=\u001b[39mflow\u001b[38;5;241m.\u001b[39mtimeout_seconds\n\u001b[1;32m 865\u001b[0m )\n\u001b[0;32m--> 867\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m flow_call\u001b[38;5;241m.\u001b[39maresult()\n\u001b[1;32m 869\u001b[0m waited_for_task_runs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m wait_for_task_runs_and_report_crashes(\n\u001b[1;32m 870\u001b[0m flow_run_context\u001b[38;5;241m.\u001b[39mtask_run_futures, client\u001b[38;5;241m=\u001b[39mclient\n\u001b[1;32m 871\u001b[0m )\n\u001b[1;32m 872\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m PausedRun \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 873\u001b[0m \u001b[38;5;66;03m# could get raised either via utility or by returning Paused from a task run\u001b[39;00m\n\u001b[1;32m 874\u001b[0m \u001b[38;5;66;03m# if a task run pauses, we set its state as the flow's state\u001b[39;00m\n\u001b[1;32m 875\u001b[0m \u001b[38;5;66;03m# to preserve reschedule and timeout behavior\u001b[39;00m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:327\u001b[0m, in \u001b[0;36mCall.aresult\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 321\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 322\u001b[0m \u001b[38;5;124;03mWait for the result of the call.\u001b[39;00m\n\u001b[1;32m 323\u001b[0m \n\u001b[1;32m 324\u001b[0m \u001b[38;5;124;03mFor use from asynchronous contexts.\u001b[39;00m\n\u001b[1;32m 325\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 326\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 327\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mwrap_future(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture)\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mCancelledError \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 329\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CancelledError() \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:352\u001b[0m, in \u001b[0;36mCall._run_sync\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 350\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39menforce_sync_deadline() \u001b[38;5;28;01mas\u001b[39;00m cancel_scope:\n\u001b[1;32m 351\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 352\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 353\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 354\u001b[0m \u001b[38;5;66;03m# Forget this call's arguments in order to free up any memory\u001b[39;00m\n\u001b[1;32m 355\u001b[0m \u001b[38;5;66;03m# that may be referenced by them; after a call has happened,\u001b[39;00m\n\u001b[1;32m 356\u001b[0m \u001b[38;5;66;03m# there's no need to keep a reference to them\u001b[39;00m\n\u001b[1;32m 357\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", "Cell \u001b[0;32mIn[4], line 26\u001b[0m, in \u001b[0;36mcombustion\u001b[0;34m(atoms)\u001b[0m\n\u001b[1;32m 6\u001b[0m future \u001b[38;5;241m=\u001b[39m MD\u001b[38;5;241m.\u001b[39msubmit(\n\u001b[1;32m 7\u001b[0m atoms\u001b[38;5;241m=\u001b[39matoms,\n\u001b[1;32m 8\u001b[0m calculator_name\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 21\u001b[0m restart\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 22\u001b[0m )\n\u001b[1;32m 24\u001b[0m futures\u001b[38;5;241m.\u001b[39mappend(future)\n\u001b[0;32m---> 26\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m[\u001b[49m\u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfuture\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfutures\u001b[49m\u001b[43m]\u001b[49m\n", "Cell \u001b[0;32mIn[4], line 26\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 6\u001b[0m future \u001b[38;5;241m=\u001b[39m MD\u001b[38;5;241m.\u001b[39msubmit(\n\u001b[1;32m 7\u001b[0m atoms\u001b[38;5;241m=\u001b[39matoms,\n\u001b[1;32m 8\u001b[0m calculator_name\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 21\u001b[0m restart\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 22\u001b[0m )\n\u001b[1;32m 24\u001b[0m futures\u001b[38;5;241m.\u001b[39mappend(future)\n\u001b[0;32m---> 26\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m futures]\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py:228\u001b[0m, in \u001b[0;36mPrefectFuture.result\u001b[0;34m(self, timeout, raise_on_failure)\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m from_async\u001b[38;5;241m.\u001b[39mcall_soon_in_loop_thread(result)\u001b[38;5;241m.\u001b[39maresult()\n\u001b[1;32m 227\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 228\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfrom_sync\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_soon_in_loop_thread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresult\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:318\u001b[0m, in \u001b[0;36mCall.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mresult\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout: Optional[\u001b[38;5;28mfloat\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m 313\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;124;03m Wait for the result of the call.\u001b[39;00m\n\u001b[1;32m 315\u001b[0m \n\u001b[1;32m 316\u001b[0m \u001b[38;5;124;03m Not safe for use from asynchronous contexts.\u001b[39;00m\n\u001b[1;32m 317\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 318\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:188\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[1;32m 187\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m--> 188\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 189\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 190\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m()\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/concurrent/futures/_base.py:401\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n\u001b[1;32m 400\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 401\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 403\u001b[0m \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:389\u001b[0m, in \u001b[0;36mCall._run_async\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 387\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39menforce_async_deadline() \u001b[38;5;28;01mas\u001b[39;00m cancel_scope:\n\u001b[1;32m 388\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 389\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 390\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 391\u001b[0m \u001b[38;5;66;03m# Forget this call's arguments in order to free up any memory\u001b[39;00m\n\u001b[1;32m 392\u001b[0m \u001b[38;5;66;03m# that may be referenced by them; after a call has happened,\u001b[39;00m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;66;03m# there's no need to keep a reference to them\u001b[39;00m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py:237\u001b[0m, in \u001b[0;36mPrefectFuture._result\u001b[0;34m(self, timeout, raise_on_failure)\u001b[0m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m final_state:\n\u001b[1;32m 236\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCall timed out before task finished.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 237\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m final_state\u001b[38;5;241m.\u001b[39mresult(raise_on_failure\u001b[38;5;241m=\u001b[39mraise_on_failure, fetch\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/states.py:91\u001b[0m, in \u001b[0;36m_get_state_result\u001b[0;34m(state, raise_on_failure)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m UnfinishedRun(\n\u001b[1;32m 85\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRun is in \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstate\u001b[38;5;241m.\u001b[39mtype\u001b[38;5;241m.\u001b[39mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m state, its result is not available.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 86\u001b[0m )\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m raise_on_failure \u001b[38;5;129;01mand\u001b[39;00m (\n\u001b[1;32m 89\u001b[0m state\u001b[38;5;241m.\u001b[39mis_crashed() \u001b[38;5;129;01mor\u001b[39;00m state\u001b[38;5;241m.\u001b[39mis_failed() \u001b[38;5;129;01mor\u001b[39;00m state\u001b[38;5;241m.\u001b[39mis_cancelled()\n\u001b[1;32m 90\u001b[0m ):\n\u001b[0;32m---> 91\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m get_state_exception(state)\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(state\u001b[38;5;241m.\u001b[39mdata, DataDocument):\n\u001b[1;32m 94\u001b[0m result \u001b[38;5;241m=\u001b[39m result_from_state_with_data_document(\n\u001b[1;32m 95\u001b[0m state, raise_on_failure\u001b[38;5;241m=\u001b[39mraise_on_failure\n\u001b[1;32m 96\u001b[0m )\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect_dask/task_runners.py:311\u001b[0m, in \u001b[0;36mDaskTaskRunner.wait\u001b[0;34m(self, key, timeout)\u001b[0m\n\u001b[1;32m 309\u001b[0m future \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_dask_future(key)\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 311\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m future\u001b[38;5;241m.\u001b[39mresult(timeout\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m distributed\u001b[38;5;241m.\u001b[39mTimeoutError:\n\u001b[1;32m 313\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/distributed/client.py:336\u001b[0m, in \u001b[0;36mFuture._result\u001b[0;34m(self, raiseit)\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m raiseit:\n\u001b[1;32m 335\u001b[0m typ, exc, tb \u001b[38;5;241m=\u001b[39m exc\n\u001b[0;32m--> 336\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m 337\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 338\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m exc\n", "\u001b[0;31mKilledWorker\u001b[0m: Attempted to run task 'md-2-9b52cfa3e22647de99659385eca3d3f2-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.66.32:43067. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html." ] } ], "source": [ "results = combustion(atoms)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "mlip-arena", "language": "python", "name": "mlip-arena" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.8" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": {}, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 4 }