Repository: carsdotcom/skelebot (language: python)

Issue:
Install from pyproject.toml
### Context
Python is making [great](https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html) [strides](https://packaging.python.org/en/latest/specifications/declaring-project-metadata/) [towards](https://stackoverflow.com/questions/69711606/how-to-install-a-package-using-pip-in-editable-mode-with-pyproject-toml) fully transitioning to a `pyproject.toml` file as a single source of configuration for package. New _package_ projects can now have all their build and dependency configuration in that file.

Since version 1.24.0 skelebot supports installing depencies from a `requirements.txt`, the accepted single source of "dependencies truth" for general python projects. However, for python packages using a pyproject.toml we have to specify  dependencies twice: in the toml and skelebot's yaml config. This will not do!

### Feature
In keeping with the DRY principle, skelebot should support reading dependencies directly from a pyproject.toml file if the user is developing a python package.

---

### Acceptance Criteria
- [x] Add toml parsing support for skelebot
- [x] Add new skelebot dependency key to read dependencies from pyproject.toml
- [x] Also allow skelebot to parse optional dependencies
- [x] Add/update unittests to maintain coverage.



Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bbc038d..fb981a0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,21 @@ Documenting All Changes to the Skelebot Project
 
 ---
 
+## v1.31.0
+#### Changed
+- **Dependencies** | Added the ability for python dependencies to be specified and installed via a `pyproject.toml` file
+
+---
+
+## v1.30.1
+#### Merged: 2022-05-24
+#### Changed
+- **CodeArtifact Dependencies** | Create libs folder if it doesn't already exist
+
+---
+
 ## v1.30.0
+#### Merged: 2022-05-04
 #### Changed
 - **Docker Publish** | Docker Publish allows for omitting version and LATEST tags
 
diff --git a/VERSION b/VERSION
index 034552a..8b96872 100644
--- a/VERSION
+++ b/VERSION
@@ -1,1 +1,1 @@
-1.30.0
+1.31.0
\ No newline at end of file
diff --git a/docs/dependencies.md b/docs/dependencies.md
index 47f8436..aebec60 100644
--- a/docs/dependencies.md
+++ b/docs/dependencies.md
@@ -23,7 +23,8 @@ Versions for packages in R can be specified by appending `={version}` to the end
 Versions for packages in Python can be specified by appending `={version}` or `=={version}` to the end of the dependency name.
 
 R and Python also both support dependencies to be installed from the local file system as well as from GitHub using the following structure.
-Python also allows for installs using a text file via `req:requirements.txt` syntax.
+
+Python also allows for installs using a text file via `req:requirements.txt` syntax or using a [`pyproject.toml`](https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html) via `proj:pyproject.toml` syntax. When using the later, all specified required and optional set(s) of dependencies will be installed.
 
 ```
 language: R
@@ -40,6 +41,7 @@ dependencies:
 - file:libs/myPackage.tgz
 - req:requirements.txt
 - github:myGitHub/fakeRepo
+- proj:pyproject.toml
 ```
 
 ### CodeArtifact Python Packages
diff --git a/requirements.txt b/requirements.txt
index 4f05c21..fda7039 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ colorama~=0.4.1
 coverage~=4.5.4
 pytest~=5.1
 boto3~=1.10
+tomli >= 1.1.0 ; python_version < "3.11"
diff --git a/skelebot-exp.yaml b/skelebot-exp.yaml
index aa4ddee..87c64e4 100644
--- a/skelebot-exp.yaml
+++ b/skelebot-exp.yaml
@@ -14,6 +14,7 @@ dependencies:
 - coverage~=4.5.4
 - pytest~=5.1
 - boto3~=1.10
+- tomli>=1.1.0
 ignores:
 - '**/*.zip'
 - '**/*.RData'
diff --git a/skelebot/systems/generators/dockerfile.py b/skelebot/systems/generators/dockerfile.py
index 71dc72a..75018c1 100644
--- a/skelebot/systems/generators/dockerfile.py
+++ b/skelebot/systems/generators/dockerfile.py
@@ -5,6 +5,12 @@
 from subprocess import call
 from ..execution import commandBuilder
 
+try:
+    import tomllib
+except ModuleNotFoundError:
+    # python <= 3.11
+    import tomli as tomllib
+
 FILE_PATH = "{path}/Dockerfile"
 
 PY_DOWNLOAD_CA = "aws codeartifact get-package-version-asset --domain {domain} --domain-owner {owner} --repository {repo} --package {pkg} --package-version {version}{profile} --format pypi --asset {asset} libs/{asset}"
@@ -33,6 +39,22 @@
 
 """
 
+def parse_pyproj(pyproject_file):
+    """Parse all required and optional dependencies from pyproject file."""
+    with open(os.path.join(os.getcwd(), pyproject_file), "rb") as f:
+        pyproj = tomllib.load(f).get("project", {})
+
+    deps = pyproj.get("dependencies", []).copy()
+    for opt_deps in pyproj.get('optional-dependencies', {}).values():
+        deps += opt_deps
+
+    # Replace any double quotes in dependencies with single quotes so we don't
+    # break the Dockerfile
+    deps = [d.replace('"', "'") for d in deps]
+
+    return '", "'.join(deps)
+
+
 def buildDockerfile(config):
     """Generates the Dockerfile based on values from the Config object"""
 
@@ -64,11 +86,15 @@ def buildDockerfile(config):
                 asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl"
                 profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else ""
                 cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile)
+                os.makedirs(os.path.join(os.getcwd(), 'libs'), exist_ok=True)
                 status = call(cmd, shell=True)
                 if (status != 0):
                     raise Exception("Failed to Obtain CodeArtifact Package")
 
                 docker += PY_INSTALL_FILE.format(depPath=f"libs/{asset}")
+            elif (dep.startswith("proj:")):
+                deps = parse_pyproj(depSplit[1])
+                docker += PY_INSTALL.format(dep=deps)
             # if using PIP version specifiers, will be handled as a standard case
             elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep):
                 verSplit = dep.split("=")
@@ -104,6 +130,7 @@ def buildDockerfile(config):
                 asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl"
                 profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else ""
                 cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile)
+                os.makedirs(os.path.join(os.getcwd(), 'libs'), exist_ok=True)
                 status = call(cmd, shell=True)
                 if (status != 0):
                     raise Exception("Failed to Obtain CodeArtifact Package")


Test patch:
diff --git a/test/files/pyproject.toml b/test/files/pyproject.toml
new file mode 100644
index 0000000..9d8f6c4
--- /dev/null
+++ b/test/files/pyproject.toml
@@ -0,0 +1,19 @@
+[project]
+name = "test"
+description = "Just a test"
+dependencies = [
+    "requests",
+    "numpy==1.15.4",
+    "pandas~=1.1",
+    'scikit-learn<=2.0.0 ; python_version<="3.6"',
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest ~= 6.2",
+    "pytest-cov ~= 3.0"
+]
+custom = [
+    "fake-package == 1.2.3",
+    "not-real"
+]
diff --git a/test/test_systems_generators_dockerfile.py b/test/test_systems_generators_dockerfile.py
index 625070e..1e93f3e 100644
--- a/test/test_systems_generators_dockerfile.py
+++ b/test/test_systems_generators_dockerfile.py
@@ -216,13 +216,14 @@ def test_buildDockerfile_cmd_path(self, mock_getcwd, mock_expanduser):
 
     @mock.patch('skelebot.systems.generators.dockerfile.call')
     @mock.patch('os.path.expanduser')
+    @mock.patch('os.makedirs')
     @mock.patch('os.getcwd')
-    def test_buildDockerfile_py_ca_file_error(self, mock_getcwd, mock_expanduser, mock_call):
+    def test_buildDockerfile_py_ca_file_error(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_call):
         folderPath = "{path}/test/files".format(path=self.path)
-        filePath = "{folder}/Dockerfile".format(folder=folderPath)
 
         mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
         mock_getcwd.return_value = folderPath
+        mock_mkdir.return_value = 1
         mock_call.return_value = 1
         config = sb.systems.generators.yaml.loadConfig()
         config.language = "Python"
@@ -236,13 +237,15 @@ def test_buildDockerfile_py_ca_file_error(self, mock_getcwd, mock_expanduser, mo
 
     @mock.patch('skelebot.systems.generators.dockerfile.call')
     @mock.patch('os.path.expanduser')
+    @mock.patch('os.makedirs')
     @mock.patch('os.getcwd')
-    def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser, mock_call):
+    def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_call):
         folderPath = "{path}/test/files".format(path=self.path)
         filePath = "{folder}/Dockerfile".format(folder=folderPath)
 
         mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
         mock_getcwd.return_value = folderPath
+        mock_mkdir.return_value = 1
         mock_call.return_value = 0
         config = sb.systems.generators.yaml.loadConfig()
         config.language = "Python"
@@ -251,6 +254,7 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser, mock_call):
         config.dependencies.append("file:libs/proj")
         config.dependencies.append("ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod")
         config.dependencies.append("req:requirements.txt")
+        config.dependencies.append("proj:pyproject.toml")
         config.dependencies.append("dtable=9.0")
 
         expectedDockerfile = """
@@ -273,6 +277,7 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser, mock_call):
 RUN ["pip", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"]
 COPY requirements.txt requirements.txt
 RUN ["pip", "install", "-r", "/app/requirements.txt"]
+RUN ["pip", "install", "requests", "numpy==1.15.4", "pandas~=1.1", "scikit-learn<=2.0.0 ; python_version<=\'3.6\'", "pytest ~= 6.2", "pytest-cov ~= 3.0", "fake-package == 1.2.3", "not-real"]
 RUN ["pip", "install", "dtable==9.0"]
 COPY . /app
 RUN rm -rf build/
@@ -286,6 +291,7 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser, mock_call):
         data = None
         with open(filePath, "r") as file:
             data = file.read()
+        mock_mkdir.assert_called_with("{folder}/libs".format(folder=folderPath), exist_ok=True)
         mock_call.assert_called_with(expectedCMD, shell=True)
         self.assertTrue(data is not None)
         self.assertEqual(data, expectedDockerfile)
@@ -483,13 +489,14 @@ def test_buildDockerfile_custom(self, mock_getcwd, mock_expanduser):
 
     @mock.patch('skelebot.systems.generators.dockerfile.call')
     @mock.patch('os.path.expanduser')
+    @mock.patch('os.makedirs')
     @mock.patch('os.getcwd')
-    def test_buildDockerfile_R_py_ca_file_error(self, mock_getcwd, mock_expanduser, mock_call):
+    def test_buildDockerfile_R_py_ca_file_error(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_call):
         folderPath = "{path}/test/files".format(path=self.path)
-        filePath = "{folder}/Dockerfile".format(folder=folderPath)
 
         mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
         mock_getcwd.return_value = folderPath
+        mock_mkdir.return_value = 1
         mock_call.return_value = -1
         config = sb.systems.generators.yaml.loadConfig()
         config.language = "R+Python"
@@ -517,13 +524,15 @@ def test_buildDockerfile_R_py_ca_file_error(self, mock_getcwd, mock_expanduser,
 
     @mock.patch('skelebot.systems.generators.dockerfile.call')
     @mock.patch('os.path.expanduser')
+    @mock.patch('os.makedirs')
     @mock.patch('os.getcwd')
-    def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser, mock_call):
+    def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_call):
         folderPath = "{path}/test/files".format(path=self.path)
         filePath = "{folder}/Dockerfile".format(folder=folderPath)
 
         mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
         mock_getcwd.return_value = folderPath
+        mock_mkdir.return_value = 1
         mock_call.return_value = 0
         config = sb.systems.generators.yaml.loadConfig()
         config.language = "R+Python"
@@ -582,6 +591,7 @@ def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser, mock_
             data = file.read()
         self.assertTrue(data is not None)
         self.assertEqual(data, expectedDockerfile)
+        mock_mkdir.assert_called_with("{folder}/libs".format(folder=folderPath), exist_ok=True)
         mock_call.assert_called_with(expectedCMD, shell=True)
 
     @mock.patch('os.path.expanduser')