Repository: CQCL/tierkreis
Pull request #48: feat: Add worker shims
Author: login: johnchildren
Created: 2025-05-06T14:46:54Z | Merged: 2025-05-12T09:48:34Z

I think I've got the shims working in a reasonable way now that passes CI (and the examples all run).

It also means that all workers that use the shims will report their errors automatically.

I think there are a couple of issues with it though:
It assumes json serialisation (I think this could be made configurable)
The shims are part of the main tierkreis dependency (maybe it could be factored out into a tierkreis-worker package?)
"glob" style inputs and outputs rely on special behaviour if there is exactly one input whose type is Iterator or an Iterator output which is slightly magical

Otherwise I think the PR is now ready for review


Diff:
diff --git a/examples/error_handling/error_handling_graph.py b/examples/error_handling/error_handling_graph.py
new file mode 100644
index 000000000..b9408e39d
--- /dev/null
+++ b/examples/error_handling/error_handling_graph.py
@@ -0,0 +1,55 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["tierkreis"]
+#
+# [tool.uv.sources]
+# tierkreis = { path = "../../tierkreis" }
+# ///
+import json
+from pathlib import Path
+from uuid import UUID
+
+from tierkreis import Labels
+from tierkreis.controller import run_graph
+from tierkreis.controller.data.graph import GraphData, Func, Output
+from tierkreis.controller.data.location import Loc
+from tierkreis.controller.storage.filestorage import ControllerFileStorage
+from tierkreis.controller.executor.uv_executor import UvExecutor
+
+root_loc = Loc()
+
+
+def error_graph() -> GraphData:
+    """A graph that errors."""
+    g = GraphData()
+
+    output = g.add(Func("error_worker.fail", {}))("value")
+
+    g.add(Output({"value": output}))
+
+    return g
+
+
+def main() -> None:
+    """Configure our workflow execution and run it to completion."""
+    # Assign a fixed uuid for our workflow.
+    workflow_id = UUID(int=103)
+    storage = ControllerFileStorage(workflow_id, name="error_handling")
+
+    # Look for workers in the same directory as this file.
+    registry_path = Path(__file__).parent
+    executor = UvExecutor(registry_path=registry_path, logs_path=storage.logs_path)
+    print("Starting workflow at location:", storage.logs_path)
+    run_graph(
+        storage,
+        executor,
+        error_graph(),
+        {Labels.VALUE: json.dumps("world!").encode()},
+        polling_interval_seconds=0.1,
+    )
+    output = storage.read_errors(root_loc)
+    print(output)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/error_handling/error_worker/main.py b/examples/error_handling/error_worker/main.py
new file mode 100644
index 000000000..a1613d71a
--- /dev/null
+++ b/examples/error_handling/error_worker/main.py
@@ -0,0 +1,34 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["pydantic", "tierkreis"]
+#
+# [tool.uv.sources]
+# tierkreis = { path = "../../../tierkreis", editable = true }
+# ///
+import logging
+from sys import argv
+from pathlib import Path
+
+
+from tierkreis import Worker, Value
+
+logger = logging.getLogger(__name__)
+
+worker = Worker("error_worker")
+
+
+@worker.function()
+def fail() -> Value[str]:
+    raise Exception("I refuse!")
+    return Value(value="I failed to refuse")
+
+
+def main() -> None:
+    node_definition_path = argv[1]
+    logger.info(node_definition_path)
+    worker.run(Path(node_definition_path))
+
+
+if __name__ == "__main__":
+    logger.info("starting worker")
+    main()
diff --git a/examples/hamiltonian/aer_worker/main.py b/examples/hamiltonian/aer_worker/main.py
index be0a485c9..88fdcdca4 100644
--- a/examples/hamiltonian/aer_worker/main.py
+++ b/examples/hamiltonian/aer_worker/main.py
@@ -1,87 +1,54 @@
 # /// script
 # requires-python = ">=3.12"
-# dependencies = ["pydantic", "pytket", "pytket-qiskit"]
+# dependencies = ["pydantic", "pytket", "pytket-qiskit", "tierkreis"]
+#
+# [tool.uv.sources]
+# tierkreis = { path = "../../../tierkreis" }
 # ///
-import json
 import logging
 from sys import argv
 from pathlib import Path
-from typing import Optional
 
 from pydantic import BaseModel
-
+from tierkreis import Worker
 from pytket._tket.circuit import Circuit
 from pytket.extensions.qiskit.backends.aer import AerBackend
 
 logger = logging.getLogger(__name__)
 
+worker = Worker("aer-worker")
 
-class NodeDefinition(BaseModel):
-    function_name: str
-    inputs: dict[str, Path]
-    outputs: dict[str, Path]
-    done_path: Path
-    logs_path: Optional[Path] = None
-
-
-def run(node_definition: NodeDefinition):
-    logging.basicConfig(
-        format="%(asctime)s: %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        filename=node_definition.logs_path,
-        filemode="a",
-        level=logging.INFO,
-    )
-    logger.info(node_definition.model_dump())
-
-    name = node_definition.function_name
 
-    if name == "submit":
-        logger.info("simulate on aer")
-        with open(node_definition.inputs["circuits"], "rb") as fh:
-            circuit_jsons = json.loads(fh.read())
-            assert isinstance(circuit_jsons, list)
-            compiled_circuits = [
-                Circuit.from_dict(x)
-                for x in circuit_jsons  # type:ignore
-            ]
-        with open(node_definition.inputs["n_shots"], "rb") as fh:
-            n_shots = json.loads(fh.read())
-            assert isinstance(n_shots, int)
+class BackendResults(BaseModel):
+    backend_results: list[dict]
 
-        backend = AerBackend()
-        results = backend.run_circuits(compiled_circuits, n_shots=n_shots)
 
-        with open(node_definition.outputs["backend_results"], "w+") as fh:
-            fh.write(json.dumps([x.to_dict() for x in results]))
+@worker.function()
+def submit(circuits: list[dict], n_shots: int) -> BackendResults:
+    pytket_circuits = [Circuit.from_dict(circuit) for circuit in circuits]
+    backend = AerBackend()
+    backend_results = backend.run_circuits(pytket_circuits, n_shots=n_shots)
+    return BackendResults(
+        backend_results=[backend_result.to_dict() for backend_result in backend_results]
+    )
 
-    elif name == "submit_single":
-        logger.info("simulate single on aer")
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit_json = json.loads(fh.read())
-            assert isinstance(circuit_json, dict)
-            compiled_circuit = Circuit.from_dict(circuit_json)
-        with open(node_definition.inputs["n_shots"], "rb") as fh:
-            n_shots = json.loads(fh.read())
-            assert isinstance(n_shots, int)
 
-        backend = AerBackend()
-        result = backend.run_circuit(compiled_circuit, n_shots=n_shots)
+class BackendResult(BaseModel):
+    backend_result: dict
 
-        with open(node_definition.outputs["backend_result"], "w+") as fh:
-            fh.write(json.dumps(result.to_dict()))
-    else:
-        raise ValueError(f"aer-worker: unknown function: {name}")
 
-    node_definition.done_path.touch()
+@worker.function()
+def submit_single(circuit: dict, n_shots: int) -> BackendResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    backend = AerBackend()
+    backend_result = backend.run_circuit(pytket_circuit, n_shots=n_shots)
+    return BackendResult(backend_result=backend_result.to_dict())
 
 
-def main():
-    logger.info("main")
+def main() -> None:
     node_definition_path = argv[1]
-    with open(node_definition_path, "r") as fh:
-        node_definition = NodeDefinition(**json.loads(fh.read()))
-    run(node_definition)
+    logger.info(node_definition_path)
+    worker.run(Path(node_definition_path))
 
 
 if __name__ == "__main__":
diff --git a/examples/hamiltonian/hamiltonian_graph.py b/examples/hamiltonian/hamiltonian_graph.py
index 78fbb1916..7b9f195b1 100644
--- a/examples/hamiltonian/hamiltonian_graph.py
+++ b/examples/hamiltonian/hamiltonian_graph.py
@@ -250,7 +250,7 @@ def main() -> None:
     ansatz = build_ansatz()
 
     # Assign a fixed uuid for our workflow.
-    workflow_id = UUID(int=0)
+    workflow_id = UUID(int=102)
     storage = ControllerFileStorage(workflow_id, name="hamiltonian")
 
     # Look for workers in the same directory as this file.
diff --git a/examples/hamiltonian/pytket_worker/main.py b/examples/hamiltonian/pytket_worker/main.py
index 7a43f689b..9ed06f159 100644
--- a/examples/hamiltonian/pytket_worker/main.py
+++ b/examples/hamiltonian/pytket_worker/main.py
@@ -1,15 +1,17 @@
 # /// script
 # requires-python = ">=3.12"
-# dependencies = ["pydantic", "pytket"]
+# dependencies = ["pydantic", "pytket", "tierkreis"]
+#
+# [tool.uv.sources]
+# tierkreis = { path = "../../../tierkreis" }
 # ///
-import json
 import logging
 from sys import argv
 from pathlib import Path
-from typing import Optional
 
 
 from pydantic import BaseModel
+from tierkreis import Worker
 from pytket.backends.backendresult import BackendResult
 from pytket._tket.circuit import Circuit
 from pytket.transform import Transform
@@ -20,96 +22,56 @@
 
 logger = logging.getLogger(__name__)
 
+worker = Worker("pytket-worker")
 
-class NodeDefinition(BaseModel):
-    function_name: str
-    inputs: dict[str, Path]
-    outputs: dict[str, Path]
-    done_path: Path
-    logs_path: Optional[Path] = None
 
+class CircuitResult(BaseModel):
+    circuit: dict
 
-def run(node_definition: NodeDefinition) -> None:
-    logging.basicConfig(
-        format="%(asctime)s: %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        filename=node_definition.logs_path,
-        filemode="a",
-        level=logging.INFO,
-    )
-    logger.info(node_definition.model_dump())
 
-    name = node_definition.function_name
-    if name == "substitute":
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit = Circuit.from_dict(json.loads(fh.read()))
+@worker.function()
+def substitute(circuit: dict, a: float, b: float, c: float) -> CircuitResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    pytket_circuit.symbol_substitution({Symbol("a"): a, Symbol("b"): b, Symbol("c"): c})
+    return CircuitResult(circuit=pytket_circuit.to_dict())
 
-        with open(node_definition.inputs["a"], "rb") as fh:
-            a = json.loads(fh.read())
 
-        with open(node_definition.inputs["b"], "rb") as fh:
-            b = json.loads(fh.read())
+@worker.function()
+def add_measure_all(circuit: dict) -> CircuitResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    pytket_circuit.measure_all()
+    return CircuitResult(circuit=pytket_circuit.to_dict())
 
-        with open(node_definition.inputs["c"], "rb") as fh:
-            c = json.loads(fh.read())
 
-        circuit.symbol_substitution({Symbol("a"): a, Symbol("b"): b, Symbol("c"): c})
+@worker.function(name="append_pauli_measurement")
+def append_pauli_measurement_impl(circuit: dict, pauli_string: list) -> CircuitResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    pytket_qubit_pauli_string = QubitPauliString.from_list(pauli_string)
+    append_pauli_measurement(pytket_qubit_pauli_string, pytket_circuit)
+    return CircuitResult(circuit=pytket_circuit.to_dict())
 
-        with open(node_definition.outputs["circuit"], "w+") as fh:
-            fh.write(json.dumps(circuit.to_dict()))
 
-    elif name == "add_measure_all":
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit = Circuit.from_dict(json.loads(fh.read()))
+@worker.function()
+def optimise_phase_gadgets(circuit: dict) -> CircuitResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    Transform.OptimisePhaseGadgets().apply(pytket_circuit)
+    return CircuitResult(circuit=pytket_circuit.to_dict())
 
-        circuit.measure_all()
 
-        with open(node_definition.outputs["circuit"], "w+") as fh:
-            fh.write(json.dumps(circuit.to_dict()))
+class ExpectationResult(BaseModel):
+    expectation: float
 
-    elif name == "append_pauli_measurement":
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit = Circuit.from_dict(json.loads(fh.read()))
 
-        with open(node_definition.inputs["pauli_string"], "rb") as fh:
-            qubit_pauli_string = QubitPauliString.from_list(json.loads(fh.read()))
-
-        append_pauli_measurement(qubit_pauli_string, circuit)
-
-        with open(node_definition.outputs["circuit"], "w+") as fh:
-            fh.write(json.dumps(circuit.to_dict()))
-
-    elif name == "optimise_phase_gadgets":
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit = Circuit.from_dict(json.loads(fh.read()))
-
-        Transform.OptimisePhaseGadgets().apply(circuit)
-
-        with open(node_definition.outputs["circuit"], "w+") as fh:
-            fh.write(json.dumps(circuit.to_dict()))
-
-    elif name == "expectation":
-        with open(node_definition.inputs["backend_result"], "rb") as fh:
-            backendresult = BackendResult.from_dict(json.loads(fh.read()))
-
-        expectation = expectation_from_counts(backendresult.get_counts())
-
-        with open(node_definition.outputs["expectation"], "w+") as fh:
-            fh.write(json.dumps(expectation))
-
-    # Also important
-    else:
-        raise ValueError(f"pytket_worker: unknown function: {name}")
-
-    # Very important!
-    node_definition.done_path.touch()
+@worker.function()
+def expectation(backend_result: dict) -> ExpectationResult:
+    result = BackendResult.from_dict(backend_result)
+    expectation = expectation_from_counts(result.get_counts())
+    return ExpectationResult(expectation=expectation)
 
 
 def main() -> None:
     node_definition_path = argv[1]
-    with open(node_definition_path, "r") as fh:
-        node_definition = NodeDefinition(**json.loads(fh.read()))
-    run(node_definition)
+    worker.run(Path(node_definition_path))
 
 
 if __name__ == "__main__":
diff --git a/examples/hello_world/hello_world_graph.py b/examples/hello_world/hello_world_graph.py
index fc7dce377..e3ab8165d 100644
--- a/examples/hello_world/hello_world_graph.py
+++ b/examples/hello_world/hello_world_graph.py
@@ -3,7 +3,7 @@
 # dependencies = ["tierkreis"]
 #
 # [tool.uv.sources]
-# tierkreis = { path = "../../tierkreis" }
+# tierkreis = { path = "../../tierkreis", editable = true }
 # ///
 import json
 from pathlib import Path
@@ -30,9 +30,9 @@ def hello_graph() -> GraphData:
     subject = g.add(Input("value"))("value")
 
     # We call the "concat" function from our worker.
-    output = g.add(Func("string_worker.concat", {"lhs": hello, "rhs": subject}))(
-        "value"
-    )
+    output = g.add(
+        Func("hello_world_worker.greet", {"greeting": hello, "subject": subject})
+    )("value")
 
     # We assign the output to the "value" label.
     g.add(Output({"value": output}))
@@ -43,8 +43,8 @@ def hello_graph() -> GraphData:
 def main() -> None:
     """Configure our workflow execution and run it to completion."""
     # Assign a fixed uuid for our workflow.
-    workflow_id = UUID(int=0)
-    storage = ControllerFileStorage(workflow_id, name="symbolic_circuits")
+    workflow_id = UUID(int=100)
+    storage = ControllerFileStorage(workflow_id, name="hello_world")
     # Clean the working directory if a prior workflow of that ID already exists
     storage.clean_graph_files()
 
diff --git a/examples/hello_world/hello_world_worker/main.py b/examples/hello_world/hello_world_worker/main.py
new file mode 100644
index 000000000..0e6d9452d
--- /dev/null
+++ b/examples/hello_world/hello_world_worker/main.py
@@ -0,0 +1,34 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["pydantic", "tierkreis"]
+#
+# [tool.uv.sources]
+# tierkreis = { path = "../../../tierkreis", editable = true }
+# ///
+import logging
+from sys import argv
+from pathlib import Path
+
+
+from tierkreis import Worker, Value
+
+logger = logging.getLogger(__name__)
+
+worker = Worker("hello_world_worker")
+
+
+@worker.function()
+def greet(greeting: str, subject: str) -> Value[str]:
+    logger.info("%s %s", greeting, subject)
+    return Value(value=greeting + subject)
+
+
+def main() -> None:
+    node_definition_path = argv[1]
+    logger.info(node_definition_path)
+    worker.run(Path(node_definition_path))
+
+
+if __name__ == "__main__":
+    logger.info("starting worker")
+    main()
diff --git a/examples/hello_world/string_worker/main.py b/examples/hello_world/string_worker/main.py
deleted file mode 100644
index 0cf3145c5..000000000
--- a/examples/hello_world/string_worker/main.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# /// script
-# requires-python = ">=3.12"
-# dependencies = ["pydantic"]
-# ///
-import json
-import logging
-from sys import argv
-from pathlib import Path
-from typing import Optional
-
-
-from pydantic import BaseModel
-
-logger = logging.getLogger(__name__)
-
-
-class NodeDefinition(BaseModel):
-    function_name: str
-    inputs: dict[str, Path]
-    outputs: dict[str, Path]
-    done_path: Path
-    error_path: Path
-    logs_path: Optional[Path] = None
-
-
-def run(node_definition: NodeDefinition) -> None:
-    logging.basicConfig(
-        format="%(asctime)s: %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        filename=node_definition.logs_path,
-        filemode="a",
-        level=logging.INFO,
-    )
-    logger.info(node_definition.model_dump())
-
-    name = node_definition.function_name
-    if name == "concat":
-        with open(node_definition.inputs["lhs"], "rb") as fh:
-            lhs = json.loads(fh.read())
-
-        with open(node_definition.inputs["rhs"], "rb") as fh:
-            rhs = json.loads(fh.read())
-
-        value = lhs + rhs
-
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(value))
-
-    # Also important
-    else:
-        node_definition.error_path.touch()
-        raise ValueError(f"string_worker: unknown function: {name}")
-
-    # Very important!
-    node_definition.done_path.touch()
-
-
-def main() -> None:
-    node_definition_path = argv[1]
-    with open(node_definition_path, "r") as fh:
-        node_definition = NodeDefinition(**json.loads(fh.read()))
-    run(node_definition)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/examples/symbolic_circuits/aer_worker/main.py b/examples/symbolic_circuits/aer_worker/main.py
index be0a485c9..88fdcdca4 100644
--- a/examples/symbolic_circuits/aer_worker/main.py
+++ b/examples/symbolic_circuits/aer_worker/main.py
@@ -1,87 +1,54 @@
 # /// script
 # requires-python = ">=3.12"
-# dependencies = ["pydantic", "pytket", "pytket-qiskit"]
+# dependencies = ["pydantic", "pytket", "pytket-qiskit", "tierkreis"]
+#
+# [tool.uv.sources]
+# tierkreis = { path = "../../../tierkreis" }
 # ///
-import json
 import logging
 from sys import argv
 from pathlib import Path
-from typing import Optional
 
 from pydantic import BaseModel
-
+from tierkreis import Worker
 from pytket._tket.circuit import Circuit
 from pytket.extensions.qiskit.backends.aer import AerBackend
 
 logger = logging.getLogger(__name__)
 
+worker = Worker("aer-worker")
 
-class NodeDefinition(BaseModel):
-    function_name: str
-    inputs: dict[str, Path]
-    outputs: dict[str, Path]
-    done_path: Path
-    logs_path: Optional[Path] = None
-
-
-def run(node_definition: NodeDefinition):
-    logging.basicConfig(
-        format="%(asctime)s: %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        filename=node_definition.logs_path,
-        filemode="a",
-        level=logging.INFO,
-    )
-    logger.info(node_definition.model_dump())
-
-    name = node_definition.function_name
 
-    if name == "submit":
-        logger.info("simulate on aer")
-        with open(node_definition.inputs["circuits"], "rb") as fh:
-            circuit_jsons = json.loads(fh.read())
-            assert isinstance(circuit_jsons, list)
-            compiled_circuits = [
-                Circuit.from_dict(x)
-                for x in circuit_jsons  # type:ignore
-            ]
-        with open(node_definition.inputs["n_shots"], "rb") as fh:
-            n_shots = json.loads(fh.read())
-            assert isinstance(n_shots, int)
+class BackendResults(BaseModel):
+    backend_results: list[dict]
 
-        backend = AerBackend()
-        results = backend.run_circuits(compiled_circuits, n_shots=n_shots)
 
-        with open(node_definition.outputs["backend_results"], "w+") as fh:
-            fh.write(json.dumps([x.to_dict() for x in results]))
+@worker.function()
+def submit(circuits: list[dict], n_shots: int) -> BackendResults:
+    pytket_circuits = [Circuit.from_dict(circuit) for circuit in circuits]
+    backend = AerBackend()
+    backend_results = backend.run_circuits(pytket_circuits, n_shots=n_shots)
+    return BackendResults(
+        backend_results=[backend_result.to_dict() for backend_result in backend_results]
+    )
 
-    elif name == "submit_single":
-        logger.info("simulate single on aer")
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit_json = json.loads(fh.read())
-            assert isinstance(circuit_json, dict)
-            compiled_circuit = Circuit.from_dict(circuit_json)
-        with open(node_definition.inputs["n_shots"], "rb") as fh:
-            n_shots = json.loads(fh.read())
-            assert isinstance(n_shots, int)
 
-        backend = AerBackend()
-        result = backend.run_circuit(compiled_circuit, n_shots=n_shots)
+class BackendResult(BaseModel):
+    backend_result: dict
 
-        with open(node_definition.outputs["backend_result"], "w+") as fh:
-            fh.write(json.dumps(result.to_dict()))
-    else:
-        raise ValueError(f"aer-worker: unknown function: {name}")
 
-    node_definition.done_path.touch()
+@worker.function()
+def submit_single(circuit: dict, n_shots: int) -> BackendResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    backend = AerBackend()
+    backend_result = backend.run_circuit(pytket_circuit, n_shots=n_shots)
+    return BackendResult(backend_result=backend_result.to_dict())
 
 
-def main():
-    logger.info("main")
+def main() -> None:
     node_definition_path = argv[1]
-    with open(node_definition_path, "r") as fh:
-        node_definition = NodeDefinition(**json.loads(fh.read()))
-    run(node_definition)
+    logger.info(node_definition_path)
+    worker.run(Path(node_definition_path))
 
 
 if __name__ == "__main__":
diff --git a/examples/symbolic_circuits/pytket_worker/main.py b/examples/symbolic_circuits/pytket_worker/main.py
index dbb1a88f8..94a6fadeb 100644
--- a/examples/symbolic_circuits/pytket_worker/main.py
+++ b/examples/symbolic_circuits/pytket_worker/main.py
@@ -1,15 +1,17 @@
 # /// script
 # requires-python = ">=3.12"
-# dependencies = ["pydantic", "pytket"]
+# dependencies = ["pydantic", "pytket", "tierkreis"]
+#
+# [tool.uv.sources]
+# tierkreis = { path = "../../../tierkreis" }
 # ///
-import json
 import logging
 from sys import argv
 from pathlib import Path
-from typing import Optional
 
 
 from pydantic import BaseModel
+from tierkreis import Worker
 from pytket.backends.backendresult import BackendResult
 from pytket._tket.circuit import Circuit
 from pytket.transform import Transform
@@ -18,84 +20,48 @@
 
 logger = logging.getLogger(__name__)
 
+worker = Worker("pytket-worker")
 
-class NodeDefinition(BaseModel):
-    function_name: str
-    inputs: dict[str, Path]
-    outputs: dict[str, Path]
-    done_path: Path
-    logs_path: Optional[Path] = None
 
+class CircuitResult(BaseModel):
+    circuit: dict
 
-def run(node_definition: NodeDefinition) -> None:
-    logging.basicConfig(
-        format="%(asctime)s: %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        filename=node_definition.logs_path,
-        filemode="a",
-        level=logging.INFO,
-    )
-    logger.info(node_definition.model_dump())
 
-    name = node_definition.function_name
-    if name == "substitute":
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit = Circuit.from_dict(json.loads(fh.read()))
+@worker.function()
+def substitute(circuit: dict, a: float, b: float, c: float) -> CircuitResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    pytket_circuit.symbol_substitution({Symbol("a"): a, Symbol("b"): b, Symbol("c"): c})
+    return CircuitResult(circuit=pytket_circuit.to_dict())
 
-        with open(node_definition.inputs["a"], "rb") as fh:
-            a = json.loads(fh.read())
 
-        with open(node_definition.inputs["b"], "rb") as fh:
-            b = json.loads(fh.read())
+@worker.function()
+def add_measure_all(circuit: dict) -> CircuitResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    pytket_circuit.measure_all()
+    return CircuitResult(circuit=pytket_circuit.to_dict())
 
-        with open(node_definition.inputs["c"], "rb") as fh:
-            c = json.loads(fh.read())
 
-        circuit.symbol_substitution({Symbol("a"): a, Symbol("b"): b, Symbol("c"): c})
+@worker.function()
+def optimise_phase_gadgets(circuit: dict) -> CircuitResult:
+    pytket_circuit = Circuit.from_dict(circuit)
+    Transform.OptimisePhaseGadgets().apply(pytket_circuit)
+    return CircuitResult(circuit=pytket_circuit.to_dict())
 
-        with open(node_definition.outputs["circuit"], "w+") as fh:
-            fh.write(json.dumps(circuit.to_dict()))
 
-    elif name == "add_measure_all":
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit = Circuit.from_dict(json.loads(fh.read()))
+class ExpectationResult(BaseModel):
+    expectation: float
 
-        circuit.measure_all()
 
-        with open(node_definition.outputs["circuit"], "w+") as fh:
-            fh.write(json.dumps(circuit.to_dict()))
-
-    elif name == "optimise_phase_gadgets":
-        with open(node_definition.inputs["circuit"], "rb") as fh:
-            circuit = Circuit.from_dict(json.loads(fh.read()))
-
-        Transform.OptimisePhaseGadgets().apply(circuit)
-
-        with open(node_definition.outputs["circuit"], "w+") as fh:
-            fh.write(json.dumps(circuit.to_dict()))
-
-    elif name == "expectation":
-        with open(node_definition.inputs["backend_result"], "rb") as fh:
-            backendresult = BackendResult.from_dict(json.loads(fh.read()))
-
-        expectation = expectation_from_counts(backendresult.get_counts())
-
-        with open(node_definition.outputs["expectation"], "w+") as fh:
-            fh.write(json.dumps(expectation))
-
-    # Also important
-    else:
-        raise ValueError(f"pytket_worker: unknown function: {name}")
-
-    # Very important!
-    node_definition.done_path.touch()
+@worker.function()
+def expectation(backend_result: dict) -> ExpectationResult:
+    result = BackendResult.from_dict(backend_result)
+    expectation = expectation_from_counts(result.get_counts())
+    return ExpectationResult(expectation=expectation)
 
 
 def main() -> None:
     node_definition_path = argv[1]
-    with open(node_definition_path, "r") as fh:
-        node_definition = NodeDefinition(**json.loads(fh.read()))
-    run(node_definition)
+    worker.run(Path(node_definition_path))
 
 
 if __name__ == "__main__":
diff --git a/examples/symbolic_circuits/symbolic_circuits.py b/examples/symbolic_circuits/symbolic_circuits.py
index e0cb43fab..9499c03df 100644
--- a/examples/symbolic_circuits/symbolic_circuits.py
+++ b/examples/symbolic_circuits/symbolic_circuits.py
@@ -89,7 +89,7 @@ def main() -> None:
     ansatz = build_ansatz()
 
     # Assign a fixed uuid for our workflow.
-    workflow_id = UUID(int=0)
+    workflow_id = UUID(int=101)
     storage = ControllerFileStorage(workflow_id, name="symbolic_circuits")
 
     # Look for workers in the same directory as this file.
diff --git a/justfile b/justfile
index c361dd14d..af1c92f90 100644
--- a/justfile
+++ b/justfile
@@ -19,4 +19,10 @@ fix:
 	{{uvrun}} ruff check --fix
 
 serve:
-	cd tierkreis_visualization && {{uvrun}} fastapi dev tierkreis_visualization/main.py
\ No newline at end of file
+	cd tierkreis_visualization && {{uvrun}} fastapi dev tierkreis_visualization/main.py
+
+examples:
+	{{uvrun}} examples/hello_world/hello_world_graph.py
+	{{uvrun}} examples/error_handling/error_handling_graph.py
+	{{uvrun}} examples/symbolic_circuits/symbolic_circuits.py
+	{{uvrun}} examples/hamiltonian/hamiltonian_graph.py
diff --git a/tierkreis/tierkreis/__init__.py b/tierkreis/tierkreis/__init__.py
index 5ae947e78..bef992d33 100644
--- a/tierkreis/tierkreis/__init__.py
+++ b/tierkreis/tierkreis/__init__.py
@@ -1,3 +1,5 @@
 from tierkreis.labels import Labels
+from tierkreis.worker import Worker
+from tierkreis.value import Value
 
-__all__ = ["Labels"]
+__all__ = ["Labels", "Worker", "Value"]
diff --git a/tierkreis/tierkreis/controller/builtins/main.py b/tierkreis/tierkreis/controller/builtins/main.py
index def9eaccb..639bfcafd 100644
--- a/tierkreis/tierkreis/controller/builtins/main.py
+++ b/tierkreis/tierkreis/controller/builtins/main.py
@@ -1,261 +1,144 @@
-import json
-import logging
-from glob import glob
 from logging import getLogger
 from pathlib import Path
 from sys import argv
-from typing import Any
+from typing import Iterator
 
 from pydantic import BaseModel
 
-logger = getLogger(__name__)
+from tierkreis.value import Value
+from tierkreis.worker import Worker
+
 
+logger = getLogger(__name__)
 
-class WorkerCallArgs(BaseModel):
-    function_name: str
-    inputs: dict[str, Path]
-    outputs: dict[str, Path]
-    output_dir: Path
-    done_path: Path
-    error_path: Path
-    logs_path: Path | None
+worker = Worker("builtins")
 
 
-def iadd(a: int, b: int) -> int:
+@worker.function()
+def iadd(*, a: int, b: int) -> Value[int]:
     logger.debug(f"iadd {a} {b}")
-    return a + b
+    return Value(value=a + b)
 
 
-def itimes(a: int, b: int) -> int:
+@worker.function()
+def itimes(*, a: int, b: int) -> Value[int]:
     logger.debug(f"itimes {a} {b}")
-    return a * b
+    return Value(value=a * b)
 
 
-def igt(a: int, b: int) -> bool:
+@worker.function()
+def igt(*, a: int, b: int) -> Value[bool]:
     logger.debug(f"igt {a} {b}")
-    return a > b
-
-
-def run(node_definition: WorkerCallArgs):
-    logging.basicConfig(
-        format="%(asctime)s: %(message)s",
-        datefmt="%Y-%m-%dT%H:%M:%S%z",
-        filename=node_definition.logs_path,
-        filemode="a",
-        level=logging.INFO,
-    )
-    logger.info(node_definition.model_dump())
-    if node_definition.function_name == "iadd":
-        with open(node_definition.inputs["a"], "rb") as fh:
-            a: int = json.loads(fh.read())
-        with open(node_definition.inputs["b"], "rb") as fh:
-            b: int = json.loads(fh.read())
-
-        c = iadd(a, b)
-
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(c))
-
-    elif node_definition.function_name == "itimes":
-        with open(node_definition.inputs["a"], "rb") as fh:
-            a: int = json.loads(fh.read())
-        with open(node_definition.inputs["b"], "rb") as fh:
-            b: int = json.loads(fh.read())
-
-        c = itimes(a, b)
-
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(c))
-
-    elif node_definition.function_name == "igt":
-        with open(node_definition.inputs["a"], "rb") as fh:
-            a: int = json.loads(fh.read())
-        with open(node_definition.inputs["b"], "rb") as fh:
-            b: int = json.loads(fh.read())
-
-        c = igt(a, b)
-
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(c))
-
-    elif node_definition.function_name == "and":
-        with open(node_definition.inputs["a"], "rb") as fh:
-            a = json.loads(fh.read())
-        with open(node_definition.inputs["b"], "rb") as fh:
-            b = json.loads(fh.read())
-
-        c = a and b
+    return Value(value=a > b)
 
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(c))
 
-    elif node_definition.function_name == "id":
-        with open(node_definition.inputs["value"], "rb") as fh:
-            value = json.loads(fh.read())
-
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(value))
-
-    elif node_definition.function_name == "fold_values":
-        values = []
-        inputs = glob(str(node_definition.inputs["values_glob"]))
-        inputs.sort(key=lambda p: int(Path(p).name))
-        for i, path in enumerate(inputs):
-            with open(path, "rb") as fh:
-                values.append(json.loads(fh.read()))
-
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(values))
-
-    elif node_definition.function_name == "unfold_values":
-        with open(node_definition.inputs["value"], "rb") as fh:
-            values_list = json.loads(fh.read())
-
-        if not isinstance(values_list, list):
-            raise ValueError("VALUE to unfold must be a list.")
-
-        for i, value in enumerate(values_list):
-            with open(node_definition.output_dir / str(i), "w+") as fh:
-                fh.write(json.dumps(value))
-
-    elif node_definition.function_name == "fold_dict":
-        values = {}
-        inputs = glob(str(node_definition.inputs["values_glob"]))
-        for path in inputs:
-            with open(path, "rb") as fh:
-                k = Path(path).name
-                values[k] = json.loads(fh.read())
-
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(values))
-
-    elif node_definition.function_name == "unfold_dict":
-        with open(node_definition.inputs["value"], "rb") as fh:
-            values_dict = json.loads(fh.read())
-
-        if not isinstance(values_dict, dict):
-            raise ValueError("VALUE to unfold must be a dict.")
-
-        for k, value in values_dict.items():
-            with open(node_definition.output_dir / k, "w+") as fh:
-                fh.write(json.dumps(value))
+@worker.function(name="and")
+def impl_and(*, a: bool, b: bool) -> Value[bool]:
+    logger.debug(f"igt {a} {b}")
+    return Value(value=a and b)
 
-    elif node_definition.function_name == "append":
-        with open(node_definition.inputs["l"], "rb") as fh:
-            list_l = json.loads(fh.read())
 
-        with open(node_definition.inputs["a"], "rb") as fh:
-            a = json.loads(fh.read())
+@worker.function(name="id")
+def impl_id[T](*, value: T) -> Value[T]:
+    logger.debug(f"id {value}")
+    return Value(value=value)
 
-        assert isinstance(list_l, list)
-        new_l: list[Any] = list_l + [a]
 
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(new_l))
+@worker.function()
+def append[T](*, l: list[T], a: T) -> Value[list[T]]:  # noqa: E741
+    l.append(a)
+    return Value(value=l)
 
-    elif node_definition.function_name == "head":
-        with open(node_definition.inputs["l"], "rb") as fh:
-            list_l = json.loads(fh.read())
 
-        assert isinstance(list_l, list)
-        head, rest = list_l[0], list_l[1:]
+class Headed[T](BaseModel):
+    head: T
+    rest: list[T]
 
-        with open(node_definition.outputs["head"], "w+") as fh:
-            fh.write(json.dumps(head))
 
-        with open(node_definition.outputs["rest"], "w+") as fh:
-            fh.write(json.dumps(rest))
+@worker.function()
+def head[T](*, l: list[T]) -> Headed[T]:  # noqa: E741
+    head, rest = l[0], l[1:]
+    return Headed(head=head, rest=rest)
 
-    elif node_definition.function_name == "len":
-        with open(node_definition.inputs["l"], "rb") as fh:
-            list_l = json.loads(fh.read())
 
-        length = len(list_l)
-        logger.info(f"LEN {length}")
+@worker.function(name="len")
+def impl_len(*, l: list) -> Value[int]:  # noqa: E741
+    logger.info("len: %s", l)
+    return Value(value=len(l))
 
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(length))
 
-    elif node_definition.function_name == "str_eq":
-        with open(node_definition.inputs["a"], "rb") as fh:
-            str_a = json.loads(fh.read())
+@worker.function()
+def str_eq(*, a: str, b: str) -> Value[bool]:
+    return Value(value=a == b)
 
-        with open(node_definition.inputs["b"], "rb") as fh:
-            str_b = json.loads(fh.read())
 
-        ans = str_a == str_b
+@worker.function()
+def str_neq(*, a: str, b: str) -> Value[bool]:
+    return Value(value=a != b)
 
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(ans))
 
-    elif node_definition.function_name == "str_neq":
-        with open(node_definition.inputs["a"], "rb") as fh:
-            str_a = json.loads(fh.read())
+@worker.function()
+def fold_values[T](values_glob: Iterator[tuple[str, T]]) -> Value[list[T]]:
+    values = [value[1] for value in values_glob]
+    return Value(value=values)
 
-        with open(node_definition.inputs["b"], "rb") as fh:
-            str_b = json.loads(fh.read())
 
-        ans = str_a != str_b
+@worker.function()
+def unfold_values[T](*, value: list[T]) -> Iterator[tuple[str, T]]:
+    for i, v in enumerate(value):
+        yield str(i), v
 
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(ans))
 
-    elif node_definition.function_name == "zip":
-        with open(node_definition.inputs["a"], "rb") as fh:
-            value_a = json.loads(fh.read())
+@worker.function()
+def fold_dict[T](values_glob: Iterator[tuple[str, T]]) -> Value[dict[str, T]]:
+    values = {k: v for k, v in values_glob}
+    return Value(value=values)
 
-        with open(node_definition.inputs["b"], "rb") as fh:
-            value_b = json.loads(fh.read())
 
-        ans = list(zip(value_a, value_b))
+@worker.function()
+def unfold_dict[T](*, value: dict[str, T]) -> Iterator[tuple[str, T]]:
+    for k, v in value.items():
+        yield k, v
 
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(ans))
 
-    elif node_definition.function_name == "unzip":
-        with open(node_definition.inputs["value"], "rb") as fh:
-            zipped: list[tuple[object, object]] = json.loads(fh.read())
+@worker.function()
+def concat(*, lhs: str, rhs: str) -> Value[str]:
+    return Value(value=lhs + rhs)
 
-        value_a, value_b = map(list, zip(*zipped))
 
-        with open(node_definition.outputs["a"], "w+") as fh:
-            fh.write(json.dumps(value_a))
+@worker.function(name="zip")
+def zip_impl[U, V](*, a: list[U], b: list[V]) -> Value[list[tuple[U, V]]]:
+    return Value(value=list(zip(a, b)))
 
-        with open(node_definition.outputs["b"], "w+") as fh:
-            fh.write(json.dumps(value_b))
 
-    elif node_definition.function_name == "tuple":
-        with open(node_definition.inputs["a"], "rb") as fh:
-            value_a = json.loads(fh.read())
+class Unzipped[U, V](BaseModel):
+    a: list[U]
+    b: list[V]
 
-        with open(node_definition.inputs["b"], "rb") as fh:
-            value_b = json.loads(fh.read())
 
-        ans = (value_a, value_b)
+@worker.function()
+def unzip[U, V](*, value: list[tuple[U, V]]) -> Unzipped[U, V]:
+    value_a, value_b = map(list, zip(*value))
+    return Unzipped(a=value_a, b=value_b)
 
-        with open(node_definition.outputs["value"], "w+") as fh:
-            fh.write(json.dumps(ans))
 
-    elif node_definition.function_name == "untuple":
-        with open(node_definition.inputs["value"], "rb") as fh:
-            value: tuple[object, object] = json.loads(fh.read())
+@worker.function(name="tuple")
+def tuple_impl[U, V](*, a: U, b: V) -> Value[tuple[U, V]]:
+    return Value(value=(a, b))
 
-        value_a, value_b = value
 
-        with open(node_definition.outputs["a"], "w+") as fh:
-            fh.write(json.dumps(value_a))
+class Untupled[U, V](BaseModel):
+    a: U
+    b: V
 
-        with open(node_definition.outputs["b"], "w+") as fh:
-            fh.write(json.dumps(value_b))
 
-    else:
-        raise ValueError(f"function name {node_definition.function_name} not found")
-    node_definition.done_path.touch()
+@worker.function()
+def untuple[U, V](*, value: tuple[U, V]) -> Untupled[U, V]:
+    logger.info("untuple: %s", value)
+    value_a, value_b = value
+    return Untupled(a=value_a, b=value_b)
 
 
 if __name__ == "__main__":
     worker_definition_path = argv[1]
-    with open(worker_definition_path, "r") as fh:
-        node_definition = WorkerCallArgs(**json.loads(fh.read()))
-
-    run(node_definition)
+    worker.run(Path(worker_definition_path))
diff --git a/tierkreis/tierkreis/value.py b/tierkreis/tierkreis/value.py
new file mode 100644
index 000000000..53639f112
--- /dev/null
+++ b/tierkreis/tierkreis/value.py
@@ -0,0 +1,10 @@
+from typing import Generic, TypeVar
+
+from pydantic import BaseModel
+
+
+T = TypeVar("T")
+
+
+class Value(BaseModel, Generic[T]):
+    value: T
diff --git a/tierkreis/tierkreis/worker.py b/tierkreis/tierkreis/worker.py
new file mode 100644
index 000000000..d7e2d00ee
--- /dev/null
+++ b/tierkreis/tierkreis/worker.py
@@ -0,0 +1,158 @@
+import json
+import logging
+from glob import glob
+from logging import getLogger
+from pathlib import Path
+from collections.abc import (
+    Iterator,
+)  # Needs to be imported from here not typing to satisfy get_origin
+from typing import Callable, Iterable, ParamSpec, TypeVar, get_origin
+
+from pydantic import BaseModel
+
+logger = getLogger(__name__)
+
+
+class WorkerCallArgs(BaseModel):
+    function_name: str
+    inputs: dict[str, Path]
+    outputs: dict[str, Path]
+    output_dir: Path
+    done_path: Path
+    error_path: Path
+    logs_path: Path | None
+
+
+Params = ParamSpec("Params")
+ReturnType = TypeVar("ReturnType", bound=BaseModel)
+
+
+def _load_args(inputs: dict[str, Path]) -> dict:
+    kwargs = {}
+    for arg_name, path in inputs.items():
+        with open(path, "rb") as fh:
+            value = json.loads(fh.read())
+
+        kwargs[arg_name] = value
+
+    return kwargs
+
+
+def _save_results(outputs: dict[str, Path], results: BaseModel) -> None:
+    for result_name, path in outputs.items():
+        with open(path, "w+") as fh:
+            fh.write(json.dumps(getattr(results, result_name)))
+
+
+def _iterable_sort_key(path_str: str) -> str | int:
+    v = Path(path_str).name
+    try:
+        return int(v)
+    except ValueError:
+        return v
+
+
+def _load_args_iterable(
+    iterable_name: str, inputs: dict[str, Path]
+) -> Iterator[tuple[str, object]]:
+    globbed_inputs = glob(str(inputs[iterable_name]))
+    globbed_inputs.sort(key=_iterable_sort_key)
+    for path in globbed_inputs:
+        name = Path(path).name
+        with open(path, "rb") as fh:
+            yield name, json.loads(fh.read())
+
+
+def _save_results_iterator(
+    output_dir: Path, results: Iterable[tuple[str, object]]
+) -> None:
+    for k, value in results:
+        with open(output_dir / k, "w+") as fh:
+            fh.write(json.dumps(value))
+
+
+class Worker:
+    functions: dict[str, Callable[[WorkerCallArgs], None]]
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.functions = {}
+
+    def function(
+        self,
+        name: str | None = None,
+    ) -> Callable[
+        [
+            Callable[
+                ...,
+                BaseModel | Iterator[tuple[str, object]],
+            ]
+        ],
+        None,
+    ]:
+        """Register a function with the worker."""
+
+        def function_decorator(
+            func: Callable[
+                ...,
+                BaseModel | Iterator[tuple[str, object]],
+            ],
+        ) -> None:
+            func_name = name if name is not None else func.__name__
+
+            def wrapper(node_definition: WorkerCallArgs):
+                annotations = func.__annotations__
+                annotations.pop("return")
+                # If there is only one argument and its value is iterator,
+                # we need to load multiple files to the iterator.
+                if (
+                    len(annotations) == 1
+                    and get_origin(next(iter(annotations.values()))) is Iterator
+                ):
+                    input_iterator_name, _ = annotations.popitem()
+                    iterator = _load_args_iterable(
+                        input_iterator_name, node_definition.inputs
+                    )
+                    results = func(iterator)
+                else:
+                    kwargs = _load_args(node_definition.inputs)
+                    results = func(**kwargs)
+
+                if isinstance(results, Iterator):
+                    _save_results_iterator(node_definition.output_dir, results)
+                else:
+                    _save_results(node_definition.outputs, results)
+
+            self.functions[func_name] = wrapper
+
+        return function_decorator
+
+    def run(self, worker_definition_path: Path) -> None:
+        """Run a function."""
+        with open(worker_definition_path, "r") as fh:
+            node_definition = WorkerCallArgs(**json.loads(fh.read()))
+
+        logging.basicConfig(
+            format="%(asctime)s: %(message)s",
+            datefmt="%Y-%m-%dT%H:%M:%S%z",
+            filename=node_definition.logs_path,
+            filemode="a",
+            level=logging.INFO,
+        )
+        logger.info(node_definition.model_dump())
+
+        try:
+            function = self.functions.get(node_definition.function_name, None)
+            if function is None:
+                raise ValueError(
+                    f"{self.name}: function name {node_definition.function_name} not found"
+                )
+            logger.info(f"running: {node_definition.function_name}")
+
+            function(node_definition)
+
+            node_definition.done_path.touch()
+        except Exception as err:
+            logger.error("encountered error: %s", err)
+            with open(node_definition.error_path, "w+") as f:
+                f.write(str(err))


Reviews:
user: login: mwpb
body: Thanks John, looks good for now (and I suppose we want to iterate a bit before end-users get their hands on it?).

* Is the intention that non-JSON inputs should just be dealt with manually as we did previously?
* In future I'm definitely keen to try to avoid end-users having to deal with the glob patterns. Might need a bit of design work first.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-12T08:25:47Z
user: login: johnchildren
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T09:14:11Z
user: login: johnchildren
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T09:15:14Z
user: login: mwpb
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-12T09:33:23Z

Review comments:
diff_hunk: @@ -0,0 +1,161 @@
+import json
+import logging
+from glob import glob
+from logging import getLogger
+from pathlib import Path
+from collections.abc import (
+    Iterator,
+)  # Needs to be imported from here not typing to satisfy get_origin
+from typing import Callable, Iterable, ParamSpec, TypeVar, get_origin
+
+from pydantic import BaseModel
+
+logger = getLogger(__name__)
+
+
+class WorkerCallArgs(BaseModel):
+    function_name: str
+    inputs: dict[str, Path]
+    outputs: dict[str, Path]
+    output_dir: Path
+    done_path: Path
+    error_path: Path
+    logs_path: Path | None
+
+
+Params = ParamSpec("Params")
+ReturnType = TypeVar("ReturnType", bound=BaseModel)
+
+
+def _load_args(inputs: dict[str, Path]) -> dict:
+    kwargs = {}
+    for arg_name, path in inputs.items():
+        if arg_name not in inputs:
path: tierkreis/tierkreis/worker.py
user: login: mwpb
body: Might be reading this wrong... but is this possible?
created_at: 2025-05-12T08:00:35Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,161 @@
+import json
+import logging
+from glob import glob
+from logging import getLogger
+from pathlib import Path
+from collections.abc import (
+    Iterator,
+)  # Needs to be imported from here not typing to satisfy get_origin
+from typing import Callable, Iterable, ParamSpec, TypeVar, get_origin
+
+from pydantic import BaseModel
+
+logger = getLogger(__name__)
+
+
+class WorkerCallArgs(BaseModel):
+    function_name: str
+    inputs: dict[str, Path]
+    outputs: dict[str, Path]
+    output_dir: Path
+    done_path: Path
+    error_path: Path
+    logs_path: Path | None
+
+
+Params = ParamSpec("Params")
+ReturnType = TypeVar("ReturnType", bound=BaseModel)
+
+
+def _load_args(inputs: dict[str, Path]) -> dict:
+    kwargs = {}
+    for arg_name, path in inputs.items():
+        if arg_name not in inputs:
+            raise ValueError(f"Required argument {arg_name} not found.")
+
+        with open(path, "rb") as fh:
+            value = json.loads(fh.read())
+
+        kwargs[arg_name] = value
+
+    return kwargs
+
+
+def _save_results(outputs: dict[str, Path], results: BaseModel) -> None:
+    for result_name, path in outputs.items():
+        with open(path, "w+") as fh:
+            fh.write(json.dumps(getattr(results, result_name)))
+
+
+def _iterable_sort_key(path_str: str) -> str | int:
+    v = Path(path_str).name
+    try:
+        return int(v)
+    except ValueError:
+        return v
+
+
+def _load_args_iterable(
+    iterable_name: str, inputs: dict[str, Path]
+) -> Iterator[tuple[str, object]]:
+    globbed_inputs = glob(str(inputs[iterable_name]))
+    globbed_inputs.sort(key=_iterable_sort_key)
+    for path in globbed_inputs:
+        name = Path(path).name
+        with open(path, "rb") as fh:
+            yield name, json.loads(fh.read())
+
+
+def _save_results_iterator(
+    output_dir: Path, results: Iterable[tuple[str, object]]
+) -> None:
+    for k, value in results:
+        with open(output_dir / k, "w+") as fh:
+            fh.write(json.dumps(value))
+
+
+class Worker:
+    functions: dict[str, Callable[[WorkerCallArgs], None]]
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.functions = {}
+
+    def function(
+        self,
+        name: str | None = None,
+    ) -> Callable[
+        [
+            Callable[
+                ...,
+                BaseModel | Iterator[tuple[str, object]],
+            ]
+        ],
+        None,
+    ]:
+        """Register a function with the worker."""
+
+        def function_decorator(
+            func: Callable[
+                ...,
+                BaseModel | Iterator[tuple[str, object]],
+            ],
+        ) -> None:
+            func_name = name if name is not None else func.__name__
+
+            def wrapper(node_definition: WorkerCallArgs):
+                annotations = func.__annotations__
+                annotations.pop("return")
+                # If there is only one argument and its value is iterator,
+                # we need to load multiple files to the iterator.
+                if (
+                    len(annotations) == 1
+                    and get_origin(next(iter(annotations.values()))) is Iterator
path: tierkreis/tierkreis/worker.py
user: login: mwpb
body: Yes indeed this is getting a little complicated! Not so much a suggestion for this PR but something to think about for the future: can we restrict use of globs to the builtins worker (where they could just be implemented by manual file management) and encourage users to use higher level functions like fold, filter etc...
created_at: 2025-05-12T08:05:00Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,161 @@
+import json
+import logging
+from glob import glob
+from logging import getLogger
+from pathlib import Path
+from collections.abc import (
+    Iterator,
+)  # Needs to be imported from here not typing to satisfy get_origin
+from typing import Callable, Iterable, ParamSpec, TypeVar, get_origin
+
+from pydantic import BaseModel
+
+logger = getLogger(__name__)
+
+
+class WorkerCallArgs(BaseModel):
+    function_name: str
+    inputs: dict[str, Path]
+    outputs: dict[str, Path]
+    output_dir: Path
+    done_path: Path
+    error_path: Path
+    logs_path: Path | None
+
+
+Params = ParamSpec("Params")
+ReturnType = TypeVar("ReturnType", bound=BaseModel)
+
+
+def _load_args(inputs: dict[str, Path]) -> dict:
+    kwargs = {}
+    for arg_name, path in inputs.items():
+        if arg_name not in inputs:
+            raise ValueError(f"Required argument {arg_name} not found.")
+
+        with open(path, "rb") as fh:
+            value = json.loads(fh.read())
+
+        kwargs[arg_name] = value
+
+    return kwargs
+
+
+def _save_results(outputs: dict[str, Path], results: BaseModel) -> None:
+    for result_name, path in outputs.items():
+        with open(path, "w+") as fh:
+            fh.write(json.dumps(getattr(results, result_name)))
+
+
+def _iterable_sort_key(path_str: str) -> str | int:
+    v = Path(path_str).name
+    try:
+        return int(v)
+    except ValueError:
+        return v
+
+
+def _load_args_iterable(
+    iterable_name: str, inputs: dict[str, Path]
+) -> Iterator[tuple[str, object]]:
+    globbed_inputs = glob(str(inputs[iterable_name]))
+    globbed_inputs.sort(key=_iterable_sort_key)
+    for path in globbed_inputs:
+        name = Path(path).name
+        with open(path, "rb") as fh:
+            yield name, json.loads(fh.read())
+
+
+def _save_results_iterator(
+    output_dir: Path, results: Iterable[tuple[str, object]]
+) -> None:
+    for k, value in results:
+        with open(output_dir / k, "w+") as fh:
+            fh.write(json.dumps(value))
+
+
+class Worker:
+    functions: dict[str, Callable[[WorkerCallArgs], None]]
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.functions = {}
+
+    def function(
+        self,
+        name: str | None = None,
+    ) -> Callable[
+        [
+            Callable[
+                ...,
+                BaseModel | Iterator[tuple[str, object]],
+            ]
+        ],
+        None,
+    ]:
+        """Register a function with the worker."""
+
+        def function_decorator(
+            func: Callable[
+                ...,
+                BaseModel | Iterator[tuple[str, object]],
+            ],
+        ) -> None:
+            func_name = name if name is not None else func.__name__
+
+            def wrapper(node_definition: WorkerCallArgs):
+                annotations = func.__annotations__
+                annotations.pop("return")
+                # If there is only one argument and its value is iterator,
+                # we need to load multiple files to the iterator.
+                if (
+                    len(annotations) == 1
+                    and get_origin(next(iter(annotations.values()))) is Iterator
+                ):
+                    input_iterator_name, _ = annotations.popitem()
+                    iterator = _load_args_iterable(
+                        input_iterator_name, node_definition.inputs
+                    )
+                    results = func(iterator)
+                else:
+                    kwargs = _load_args(node_definition.inputs)
+                    results = func(**kwargs)
+
+                if isinstance(results, Iterator):
+                    _save_results_iterator(node_definition.output_dir, results)
+                else:
path: tierkreis/tierkreis/worker.py
user: login: mwpb
body: Maybe one way around the JSON assumption is to deal with non-JSONable types here. I.e. if we see a `bytes` annotation just return the raw bytes.
created_at: 2025-05-12T08:08:03Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,161 @@
+import json
+import logging
+from glob import glob
+from logging import getLogger
+from pathlib import Path
+from collections.abc import (
+    Iterator,
+)  # Needs to be imported from here not typing to satisfy get_origin
+from typing import Callable, Iterable, ParamSpec, TypeVar, get_origin
+
+from pydantic import BaseModel
+
+logger = getLogger(__name__)
+
+
+class WorkerCallArgs(BaseModel):
+    function_name: str
+    inputs: dict[str, Path]
+    outputs: dict[str, Path]
+    output_dir: Path
+    done_path: Path
+    error_path: Path
+    logs_path: Path | None
+
+
+Params = ParamSpec("Params")
+ReturnType = TypeVar("ReturnType", bound=BaseModel)
+
+
+def _load_args(inputs: dict[str, Path]) -> dict:
+    kwargs = {}
+    for arg_name, path in inputs.items():
+        if arg_name not in inputs:
+            raise ValueError(f"Required argument {arg_name} not found.")
+
+        with open(path, "rb") as fh:
+            value = json.loads(fh.read())
+
+        kwargs[arg_name] = value
+
+    return kwargs
+
+
+def _save_results(outputs: dict[str, Path], results: BaseModel) -> None:
+    for result_name, path in outputs.items():
+        with open(path, "w+") as fh:
+            fh.write(json.dumps(getattr(results, result_name)))
+
+
+def _iterable_sort_key(path_str: str) -> str | int:
+    v = Path(path_str).name
+    try:
+        return int(v)
+    except ValueError:
+        return v
+
+
+def _load_args_iterable(
+    iterable_name: str, inputs: dict[str, Path]
+) -> Iterator[tuple[str, object]]:
+    globbed_inputs = glob(str(inputs[iterable_name]))
+    globbed_inputs.sort(key=_iterable_sort_key)
+    for path in globbed_inputs:
+        name = Path(path).name
+        with open(path, "rb") as fh:
+            yield name, json.loads(fh.read())
+
+
+def _save_results_iterator(
+    output_dir: Path, results: Iterable[tuple[str, object]]
+) -> None:
+    for k, value in results:
+        with open(output_dir / k, "w+") as fh:
+            fh.write(json.dumps(value))
+
+
+class Worker:
+    functions: dict[str, Callable[[WorkerCallArgs], None]]
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.functions = {}
+
+    def function(
+        self,
+        name: str | None = None,
+    ) -> Callable[
+        [
+            Callable[
+                ...,
+                BaseModel | Iterator[tuple[str, object]],
+            ]
+        ],
+        None,
+    ]:
+        """Register a function with the worker."""
+
+        def function_decorator(
+            func: Callable[
+                ...,
+                BaseModel | Iterator[tuple[str, object]],
+            ],
+        ) -> None:
+            func_name = name if name is not None else func.__name__
+
+            def wrapper(node_definition: WorkerCallArgs):
+                annotations = func.__annotations__
+                annotations.pop("return")
+                # If there is only one argument and its value is iterator,
+                # we need to load multiple files to the iterator.
+                if (
+                    len(annotations) == 1
+                    and get_origin(next(iter(annotations.values()))) is Iterator
+                ):
+                    input_iterator_name, _ = annotations.popitem()
+                    iterator = _load_args_iterable(
+                        input_iterator_name, node_definition.inputs
+                    )
+                    results = func(iterator)
+                else:
+                    kwargs = _load_args(node_definition.inputs)
+                    results = func(**kwargs)
+
+                if isinstance(results, Iterator):
+                    _save_results_iterator(node_definition.output_dir, results)
+                else:
path: tierkreis/tierkreis/worker.py
user: login: johnchildren
body: Ah yes, sorry I didn't read this before commenting but that would make sense here I think.
created_at: 2025-05-12T09:14:11Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,161 @@
+import json
+import logging
+from glob import glob
+from logging import getLogger
+from pathlib import Path
+from collections.abc import (
+    Iterator,
+)  # Needs to be imported from here not typing to satisfy get_origin
+from typing import Callable, Iterable, ParamSpec, TypeVar, get_origin
+
+from pydantic import BaseModel
+
+logger = getLogger(__name__)
+
+
+class WorkerCallArgs(BaseModel):
+    function_name: str
+    inputs: dict[str, Path]
+    outputs: dict[str, Path]
+    output_dir: Path
+    done_path: Path
+    error_path: Path
+    logs_path: Path | None
+
+
+Params = ParamSpec("Params")
+ReturnType = TypeVar("ReturnType", bound=BaseModel)
+
+
+def _load_args(inputs: dict[str, Path]) -> dict:
+    kwargs = {}
+    for arg_name, path in inputs.items():
+        if arg_name not in inputs:
path: tierkreis/tierkreis/worker.py
user: login: johnchildren
body: I think this might be a mistake from an earlier version of this code where I was comparing the function signature to the inputs that were provided to the worker, but as you say this is now impossible, I will remove.
created_at: 2025-05-12T09:15:14Z
author_association: COLLABORATOR

PR comments:
user: login: johnchildren
created_at: 2025-05-12T09:11:16Z
body: > Thanks John, looks good for now (and I suppose we want to iterate a bit before end-users get their hands on it

I think so, it's still quite rough and I think there are probably a few low hanging improvements. For now I'm mostly keen to just get it working minimally as having the error handling by default is quite useful for authoring and debugging workflows.

>     * Is the intention that non-JSON inputs should just be dealt with manually as we did previously?

I have a couple of alternative ideas here, let me know if you have a preference:

* We could make the user provide a serialiser/deserialiser when instantiating `Worker` like:
  ```python
  worker = Worker("my-worker", serialiser=json.dumps, deserialiser=msgpack.loads)
  ```
  Which could also be override on a per function level
* We add more special types that allow a lower level control, like
  ```python
    @worker.function(output_files={"out"})
    def my_special_function(*, a: File, out: File):
        pass
  ```
  Where the "out" file is opened in write mode.
* We special case `bytes` so that if a type in a function is bytes we do zero serialization/deserialization.

>     * In future I'm definitely keen to try to avoid end-users having to deal with the glob patterns. Might need a bit of design work first.

Yeah, it's worth thinking about. I do actually kind of like the iterator-y access mode but it's a bit too complicated for regular users perhaps.


author_association: COLLABORATOR
user: login: mwpb
created_at: 2025-05-12T09:34:50Z
body: All three seem reasonable, although for the second we might have to abstract the storage interface a bit.
author_association: COLLABORATOR