Repository: AgnostiqHQ/covalent (language: python)

Issue:
Add C Support
### What should we add?

Allow C functions to be called as electrons

### Describe alternatives you've considered.

_No response_


Discussion / hints:

Fix patch:
diff --git a/.flake8 b/.flake8
index 737cbd690..d2283b44d 100644
--- a/.flake8
+++ b/.flake8
@@ -20,6 +20,6 @@
 
 [flake8]
 max-line-length = 99
-max-complexity = 18
+max-complexity = 25
 select = B,C,E,F,W,T4,B9
 ignore = E501, E722, W503, F401, F403, F811, F841, E203
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d320bc15f..6441586a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.28.0] - 2022-03-02
+
+### Added
+
+- Lepton support, including for Python modules and C libraries
+- How-to guides showing how to use leptons for each of these
+
 ## [0.27.6] - 2022-03-01
 
 ### Added
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..c26b4bc13
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,5 @@
+all:
+	gcc -shared -fPIC -o libtest.so test.c
+
+clean:
+	rm -f test.so
diff --git a/VERSION b/VERSION
index c7c49b4b4..697f087f3 100644
--- a/VERSION
+++ b/VERSION
@@ -1,1 +1,1 @@
-0.27.6
+0.28.0
diff --git a/covalent/__init__.py b/covalent/__init__.py
index b57e979fd..5e97f6ef8 100644
--- a/covalent/__init__.py
+++ b/covalent/__init__.py
@@ -28,7 +28,7 @@
 from ._results_manager.results_manager import cancel, get_result, sync
 from ._shared_files.config import get_config, reload_config, set_config
 from ._shared_files.util_classes import RESULT_STATUS as status
-from ._workflow import electron, lattice
+from ._workflow import Lepton, electron, lattice
 
 try:
     with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../VERSION")) as f:
diff --git a/covalent/_shared_files/defaults.py b/covalent/_shared_files/defaults.py
index 78128eb6e..ab7054eab 100644
--- a/covalent/_shared_files/defaults.py
+++ b/covalent/_shared_files/defaults.py
@@ -31,6 +31,7 @@
 generator_prefix = f"{prefix_separator}generated{prefix_separator}"
 sublattice_prefix = f"{prefix_separator}sublattice{prefix_separator}"
 attr_prefix = f"{prefix_separator}attribute{prefix_separator}"
+arg_prefix = f"{prefix_separator}arg{prefix_separator}"
 
 # Default configuration settings
 _DEFAULT_CONFIG = {
diff --git a/covalent/_workflow/__init__.py b/covalent/_workflow/__init__.py
index 9927b3a6c..8718dbef8 100644
--- a/covalent/_workflow/__init__.py
+++ b/covalent/_workflow/__init__.py
@@ -22,3 +22,4 @@
 
 from .electron import electron
 from .lattice import lattice
+from .lepton import Lepton
diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py
index 624a1cc7e..cedafcb8f 100644
--- a/covalent/_workflow/electron.py
+++ b/covalent/_workflow/electron.py
@@ -29,6 +29,7 @@
 from .._shared_files.context_managers import active_lattice_manager
 from .._shared_files.defaults import (
     _DEFAULT_CONSTRAINT_VALUES,
+    arg_prefix,
     attr_prefix,
     electron_dict_prefix,
     electron_list_prefix,
diff --git a/covalent/_workflow/lepton.py b/covalent/_workflow/lepton.py
new file mode 100644
index 000000000..f25a59410
--- /dev/null
+++ b/covalent/_workflow/lepton.py
@@ -0,0 +1,211 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the GNU Affero General Public License 3.0 (the "License").
+# A copy of the License may be obtained with this software package or at
+#
+#      https://www.gnu.org/licenses/agpl-3.0.en.html
+#
+# Use of this file is prohibited except in compliance with the License. Any
+# modifications or derivative works of this file must retain this copyright
+# notice, and modified files must contain a notice indicating that they have
+# been altered from the originals.
+#
+# Covalent is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
+#
+# Relief from the License may be granted by purchasing a commercial license.
+
+"""Language translation module for Electron objects."""
+
+from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union
+
+from .._shared_files import logger
+from .._shared_files.defaults import _DEFAULT_CONSTRAINT_VALUES
+from .electron import Electron
+
+if TYPE_CHECKING:
+    from ..executor import BaseExecutor
+
+app_log = logger.app_log
+log_stack_info = logger.log_stack_info
+
+
+class Lepton(Electron):
+    """
+    A generalization of an Electron to languages other than Python.
+
+    Leptons inherit from Electrons, overloading the `function` attribute
+    with a wrapper function. Users specify the foreign function's signature
+    as well as its location by providing a library and entrypoint. When one
+    of the executors invokes the task's `function`, the foreign function
+    is called by way of the wrapper function defined here. If compilation
+    scripts are required, these must be separately copied to the backend.
+
+    Attributes:
+        language: Language in which the task specification is written.
+        library_name: Name of the library or module which specifies the function.
+        function_name: Name of the foreign function.
+        argtypes: List of tuples specifying data types and input/output properties.
+    """
+
+    INPUT = 0
+    OUTPUT = 1
+    INPUT_OUTPUT = 2
+
+    _LANG_PY = ["Python", "python"]
+    _LANG_C = ["C", "c"]
+
+    def __init__(
+        self,
+        language: str = "python",
+        library_name: str = "",
+        function_name: str = "",
+        argtypes: Optional[List] = [],
+        *,
+        executor: Union[
+            List[Union[str, "BaseExecutor"]], Union[str, "BaseExecutor"]
+        ] = _DEFAULT_CONSTRAINT_VALUES["executor"],
+    ) -> None:
+        self.language = language
+        self.library_name = library_name
+        self.function_name = function_name
+        # Types must be stored as strings, since not all type objects can be pickled
+        self.argtypes = [(arg[0].__name__, arg[1]) for arg in argtypes]
+
+        # Assign the wrapper below as the task's callable function
+        super().__init__(self.wrap_task())
+
+        # Assign metadata defaults
+        super().set_metadata("executor", executor)
+
+    def wrap_task(self) -> Callable:
+        """Return a lepton wrapper function."""
+
+        def python_wrapper(*args, **kwargs) -> Any:
+            """Call a Python function specified in some other module."""
+
+            import importlib
+
+            try:
+                module = importlib.import_module(self.library_name)
+            except ModuleNotFoundError:
+                app_log.warning(f"Could not import the module '{self.library_name}'.")
+                raise
+
+            try:
+                func = getattr(module, self.function_name)
+            except AttributeError:
+                app_log.warning(
+                    f"Could not find the function '{self.function_name}' in '{self.library_name}'."
+                )
+                raise
+
+            # Foreign function invoked
+            return func(*args, **kwargs)
+
+        def c_wrapper(*args, **kwargs) -> Any:
+            """Call a C function specified in a shared library."""
+
+            import ctypes
+
+            if kwargs:
+                raise ValueError(
+                    f"Keyword arguments {kwargs} are not supported when calling {self.function}."
+                )
+
+            try:
+                handle = ctypes.CDLL(self.library_name)
+            except OSError:
+                app_log.warning(f"Could not open '{self.library_name}'.")
+                raise
+
+            # Format the variable type translation
+            entrypoint = self.function_name
+            types = []
+            for t in self.argtypes:
+                if t[0].startswith("LP_"):  # This is a pointer
+                    types.append(ctypes.POINTER(getattr(ctypes, t[0][3:])))
+                else:
+                    types.append(getattr(ctypes, t[0]))
+            attrs = [a[1] for a in self.argtypes]
+            handle[entrypoint].argtypes = types
+            handle[entrypoint].restype = None
+
+            # Translate the variables
+            c_func_args = []
+            for idx, t in enumerate(types):
+                arg = args[idx] if attrs[idx] != Lepton.OUTPUT else None
+
+                # 1. The user specifies a scalar (non-subscriptable)
+                #    and this variable returns data. It is transformed
+                #    to a pointer and passed by address.
+                if (
+                    attrs[idx] != Lepton.INPUT
+                    and not hasattr(arg, "__getitem__")
+                    and types[idx].__name__.startswith("LP_")
+                ):
+                    if arg:
+                        # The variable is used for input and output
+                        c_func_args.append(ctypes.pointer(types[idx]._type_(arg)))
+                    else:
+                        # The variable is used for output only
+                        c_func_args.append(ctypes.pointer(types[idx]._type_()))
+
+                # 2. The user specifies an array (subscriptable)
+                elif hasattr(arg, "__getitem__") and types[idx].__name__.startswith("LP_"):
+                    c_func_args.append((types[idx]._type_ * len(arg))(*arg))
+
+                # 3. Arg passed by value
+                elif attrs[idx] == self.INPUT:
+                    c_func_args.append(types[idx](arg))
+
+                else:
+                    raise ValueError("An invalid type was specified.")
+
+            # Foreign function invoked
+            handle[entrypoint](*c_func_args)
+
+            # Format the return values
+            return_vals = []
+            for idx, arg in enumerate(c_func_args):
+                if attrs[idx] == self.INPUT:
+                    continue
+
+                # 1. Convert pointers to scalars
+                if hasattr(arg, "contents"):
+                    return_vals.append(arg.contents.value)
+
+                # 2. This is a subscriptable object
+                elif hasattr(arg, "__getitem__"):
+                    return_vals.append([x.value for x in arg])
+
+                # If we end up here, the previous if/else block needs improving
+                else:
+                    raise ValueError("An invalid return type was encountered.")
+
+            if not return_vals:
+                return None
+            elif len(return_vals) == 1:
+                return return_vals[0]
+            else:
+                return tuple(return_vals)
+
+        if self.language in Lepton._LANG_PY:
+            wrapper = python_wrapper
+        elif self.language in Lepton._LANG_C:
+            wrapper = c_wrapper
+        else:
+            raise ValueError(f"Language '{self.language}' is not supported.")
+
+        # Attribute translation
+        wrapper.__name__ = self.function_name
+        wrapper.__qualname__ = f"Lepton.{self.library_name.split('.')[0]}.{self.function_name}"
+        wrapper.__module__ += f".{self.library_name.split('.')[0]}"
+        wrapper.__doc__ = (
+            f"""Lepton interface for {self.language} function '{self.function_name}'."""
+        )
+
+        return wrapper
diff --git a/doc/source/how_to/index.rst b/doc/source/how_to/index.rst
index ad484d016..57fd5789b 100644
--- a/doc/source/how_to/index.rst
+++ b/doc/source/how_to/index.rst
@@ -9,10 +9,12 @@ These are practical step-by-step guides to help you achieve a specific goal. The
 Orchestration
 *******************
 - :doc:`How to construct an electron <./orchestration/construct_electron>`
+- :doc:`How to construct a lepton <./orchestration/construct_lepton>`
 - :doc:`How to construct a lattice <./orchestration/construct_lattice>`
 - :doc:`How to add an electron to lattice <./orchestration/add_electron_to_lattice>`
 - :doc:`How to visualize the lattice <./orchestration/visualize_lattice>`
 - :doc:`How to add constraints to lattices <./orchestration/add_constraints_to_lattice>`
+- :doc:`How to construct leptons using C code <./orchestration/construct_c_task>`
 
 
 Execution
diff --git a/doc/source/how_to/orchestration/construct_c_task.ipynb b/doc/source/how_to/orchestration/construct_c_task.ipynb
new file mode 100644
index 000000000..118a05b26
--- /dev/null
+++ b/doc/source/how_to/orchestration/construct_c_task.ipynb
@@ -0,0 +1,192 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0aa3c4c0",
+   "metadata": {},
+   "source": [
+    "## How to construct tasks from C code\n",
+    "\n",
+    "Often we find workflows consist of different functions written in different languages, each most suitable for the task being executed. This example demonstrates how to call a compiled C function as an `electron`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "f45b9a78",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import covalent as ct\n",
+    "from ctypes import POINTER, c_int32"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a6c2ec0b",
+   "metadata": {},
+   "source": [
+    "First, let's write a simple program in C:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "95422edc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "c_source = \"\"\"\n",
+    "#include \"test.h\"\n",
+    "\n",
+    "void test_entry(int x, int *y, int *z)\n",
+    "{\n",
+    "        *y += x;\n",
+    "        *z = 5;\n",
+    "}\n",
+    "\"\"\"\n",
+    "\n",
+    "c_header = \"\"\"\n",
+    "void test_entry(int x, int *y, int *z);\n",
+    "\"\"\"\n",
+    "\n",
+    "with open(\"test.c\", \"w\") as f:\n",
+    "    f.write(c_source)\n",
+    "    \n",
+    "with open(\"test.h\", \"w\") as f:\n",
+    "    f.write(c_header)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d87a5fa8",
+   "metadata": {},
+   "source": [
+    "Next, compile it into a shared library:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5ffb1624",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!gcc -shared -fPIC -o libtest.so test.c"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4086fb68",
+   "metadata": {},
+   "source": [
+    "Optionally, confirm the entrypoint function is exposed in the library:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "2b97d0c7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0000000000001129 T test_entry\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "!nm -D libtest.so | grep 'T test_entry'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b3d74cb2",
+   "metadata": {},
+   "source": [
+    "Now, we are ready to construct a task which interfaces with the compiled function using a Lepton object. Note the last argument which helps the Lepton understand how to convert types."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "d23fe762",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "task = ct.Lepton(\n",
+    "    \"C\",\n",
+    "    \"libtest.so\",\n",
+    "    \"test_entry\",\n",
+    "    [\n",
+    "        (c_int32, ct.Lepton.INPUT),\n",
+    "        (POINTER(c_int32), ct.Lepton.INPUT_OUTPUT),\n",
+    "        (POINTER(c_int32), ct.Lepton.OUTPUT)\n",
+    "    ]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d14e3ce4",
+   "metadata": {},
+   "source": [
+    "Finally, use the Lepton in the context of a lattice:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "7826e8cd",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(3, 5)\n"
+     ]
+    }
+   ],
+   "source": [
+    "@ct.lattice\n",
+    "def workflow(x: int, y: int) -> int:\n",
+    "    return task(x, y)\n",
+    "\n",
+    "result = ct.dispatch_sync(workflow)(1, 2)\n",
+    "print(result.result)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bd101ade",
+   "metadata": {},
+   "source": [
+    "Note the return values consist of all input-output and output-only variables. Output-only variables can only be scalars, since the length is not otherwise known. When you want to return an array, declare it as an input-output variable and initialize it appropriately before passing it to the lepton."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/doc/source/how_to/orchestration/construct_lepton.ipynb b/doc/source/how_to/orchestration/construct_lepton.ipynb
new file mode 100644
index 000000000..4e06d12bf
--- /dev/null
+++ b/doc/source/how_to/orchestration/construct_lepton.ipynb
@@ -0,0 +1,149 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2bd6c1d8",
+   "metadata": {},
+   "source": [
+    "## How to construct a lepton\n",
+    "\n",
+    "An lepton is a generalization of an electron (task). These are used instead of electrons when the task definition lives somewhere else, such as a Python module or a compiled C library.  They are particularly useful for workflows comprised of tasks in a variety of languages. In this example we demonstrate how to construct a Lepton provided a task definition in a Python module."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "0b6d963d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import covalent as ct"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7bbbac07",
+   "metadata": {},
+   "source": [
+    "First create an example module:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f9d80c4e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_module = \"\"\"\n",
+    "def entrypoint(x: int, y: int) -> int:\n",
+    "    return x + y\n",
+    "\"\"\"\n",
+    "\n",
+    "with open(\"test_module.py\", \"w\") as f:\n",
+    "    f.write(test_module)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5e07c88d",
+   "metadata": {},
+   "source": [
+    "Create the lepton object:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "e4419fdd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "task = ct.Lepton(\n",
+    "    \"python\",\n",
+    "    \"test_module\",\n",
+    "    \"entrypoint\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "489e84ec",
+   "metadata": {},
+   "source": [
+    "The task definition can now be used in a workflow as if it were an `electron` object:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "17a96356",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@ct.lattice\n",
+    "def workflow(x: int) -> int:\n",
+    "    return task(x, 1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "da2d1e66",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Lattice Result\n",
+      "==============\n",
+      "status: COMPLETED\n",
+      "result: 3\n",
+      "inputs: {'args': [2], 'kwargs': {}}\n",
+      "error: None\n",
+      "\n",
+      "start_time: 2022-03-02 04:09:54.939862+00:00\n",
+      "end_time: 2022-03-02 04:09:54.968830+00:00\n",
+      "\n",
+      "results_dir: /home/user/covalent/results\n",
+      "dispatch_id: 4716fced-5aa6-4375-8208-ecaf13bca2ac\n",
+      "\n",
+      "Node Outputs\n",
+      "------------\n",
+      "entrypoint(0): 3\n",
+      ":parameter:2(1): 2\n",
+      ":parameter:1(2): 1\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = ct.dispatch_sync(workflow)(2)\n",
+    "print(result)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}


Test patch:
diff --git a/tests/functional_tests/docs_how_to_test.py b/tests/functional_tests/docs_how_to_test.py
index eca5b43aa..5fa595562 100644
--- a/tests/functional_tests/docs_how_to_test.py
+++ b/tests/functional_tests/docs_how_to_test.py
@@ -32,8 +32,9 @@
 files = glob.glob(rootdir + how_to_dir + suffix, recursive=True)
 expected_return_values = [0] * len(files)
 
-# Skip these since they require graphviz
+# Skip these since they require graphviz and/or gcc
 ignore_files = [
+    "construct_c_task.ipynb",
     "query_electron_execution_status.ipynb",
     "query_lattice_execution_status.ipynb",
     "visualize_lattice.ipynb",