Repository: getindata/data-pipelines-cli (language: python)

Issue:
Add `init` and `create` commands
---
Keep in mind:
- [X] Documentation updates
- [X] [Changelog](CHANGELOG.md) updates


Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cd230ad..23b6d18 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,3 +4,4 @@
 
 ### Added
 - Draft of `dp init`, `dp create`, `dp template new`, `dp template list` and `dp dbt`
+- Draft of `dp compile` and `dp deploy`
diff --git a/data_pipelines_cli/cli.py b/data_pipelines_cli/cli.py
index c96b54b..db1085e 100644
--- a/data_pipelines_cli/cli.py
+++ b/data_pipelines_cli/cli.py
@@ -1,32 +1,11 @@
-import getpass
-import pathlib
-import shutil
-import subprocess
-import sys
-from typing import Dict
-
 import click
-import copier
-import questionary
-import yaml
-
-if sys.version_info >= (3, 8):
-    from typing import TypedDict  # pylint: disable=no-name-in-module
-else:
-    from typing_extensions import TypedDict
-
-
-CONFIGURATION_PATH: pathlib.Path = pathlib.Path.home().joinpath(".dp.yml")
-
-
-class TemplateConfig(TypedDict):
-    template_name: str
-    template_path: str
-
 
-class DataPipelinesConfig(TypedDict):
-    username: str
-    templates: Dict[str, TemplateConfig]
+from .cli_commands.compile import compile_project_command
+from .cli_commands.create import create_command
+from .cli_commands.dbt import dbt_command
+from .cli_commands.deploy import deploy_command
+from .cli_commands.init import init_command
+from .cli_commands.template import template_group
 
 
 @click.group()
@@ -34,120 +13,9 @@ def cli() -> None:
     pass
 
 
-@cli.command()
-def init() -> None:
-    if CONFIGURATION_PATH.is_file():
-        overwrite_confirm = questionary.confirm(
-            "dp config already exists. Do you want to overwrite it?",
-            default=False,
-        ).ask()
-        if not overwrite_confirm:
-            sys.exit(1)
-
-    username: str = questionary.text(
-        "What's your username?",
-        default=getpass.getuser(),
-        validate=lambda text: len(text) > 0,
-    ).ask()
-
-    config: DataPipelinesConfig = {
-        "username": username,
-        "templates": {},
-    }
-    with open(CONFIGURATION_PATH, "w") as f:
-        yaml.dump(config, f, default_flow_style=False)
-
-
-def read_config_or_exit() -> DataPipelinesConfig:
-    if not CONFIGURATION_PATH.is_file():
-        click.secho(
-            "No configuration file found. Run 'dp init' to create it.",
-            fg="red",
-        )
-        sys.exit(1)
-
-    with open(CONFIGURATION_PATH, "r") as f:
-        return yaml.safe_load(f)
-
-
-@cli.group()
-def template() -> None:
-    pass
-
-
-@template.command(name="new")
-def new_template() -> None:
-    config = read_config_or_exit()
-    template_name = questionary.text(
-        "Name of your project template", validate=lambda text: len(text) > 0
-    ).ask()
-    if config["templates"].get(template_name):
-        overwrite_confirm = questionary.confirm(
-            f"Project template named {template_name} already exists. "
-            f"Do you want to overwrite it?",
-            default=False,
-        ).ask()
-        if not overwrite_confirm:
-            sys.exit(1)
-    template_path: str = questionary.text(
-        "Template path (repository address)",
-        validate=lambda text: len(text) > 0,
-    ).ask()
-
-    config["templates"][template_name] = {
-        "template_name": template_name,
-        "template_path": template_path,
-    }
-
-    with open(CONFIGURATION_PATH, "w") as f:
-        yaml.dump(config, f, default_flow_style=False)
-
-
-@template.command(name="list")
-def list_templates() -> None:
-    config = read_config_or_exit()
-
-    click.echo("AVAILABLE TEMPLATES:\n")
-    for tc in config["templates"].values():
-        click.echo(yaml.dump(tc))
-
-
-@cli.command()
-@click.argument(
-    "project_path",
-    type=click.Path(
-        writable=True, path_type=str, dir_okay=True, file_okay=False
-    ),
-)
-def create(project_path: str) -> None:
-    config = read_config_or_exit()
-    if len(config["templates"]) == 0:
-        click.secho(
-            "No template provided. Run 'dp template new' to define one.",
-            fg="red",
-        )
-        sys.exit(1)
-
-    template_name = questionary.select(
-        "", choices=list(config["templates"].keys())
-    ).ask()
-    template_config = config["templates"][template_name]
-    copier.copy(
-        src_path=template_config["template_path"], dst_path=project_path
-    )
-
-
-@cli.command()
-@click.argument("command")
-@click.option("-t", "--target", default="local", type=str)
-def dbt(command: str, target: str) -> None:
-    if not shutil.which("dbt"):
-        click.secho(
-            "dbt not found. Install dbt "
-            "[https://docs.getdbt.com/dbt-cli/installation] "
-            "and rerun the command",
-            fg="red",
-        )
-        sys.exit(1)
-
-    subprocess.run(["dbt", command, "--profiles-dir", ".", "--target", target])
+cli.add_command(init_command)
+cli.add_command(template_group)
+cli.add_command(create_command)
+cli.add_command(dbt_command)
+cli.add_command(compile_project_command)
+cli.add_command(deploy_command)
diff --git a/data_pipelines_cli/cli_commands/__init__.py b/data_pipelines_cli/cli_commands/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/data_pipelines_cli/cli_commands/compile.py b/data_pipelines_cli/cli_commands/compile.py
new file mode 100644
index 0000000..ec72dd9
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/compile.py
@@ -0,0 +1,146 @@
+import pathlib
+import shutil
+import sys
+from typing import Optional
+
+import click
+
+from ..cli_constants import (
+    BUILD_DIR,
+    DATAHUB_URL_ENV,
+    DOCKER_REPOSITORY_URL_TO_REPLACE,
+    IMAGE_TAG_TO_REPLACE,
+    INGEST_ENDPOINT_TO_REPLACE,
+)
+from ..cli_utils import (
+    echo_error,
+    echo_info,
+    echo_subinfo,
+    get_argument_or_environment_variable,
+)
+from ..data_structures import DockerArgs
+from ..io_utils import replace
+from .dbt import dbt
+
+
+def _replace_image_tag(k8s_config: pathlib.Path, docker_args: DockerArgs) -> None:
+    echo_info(f"Replacing <IMAGE_TAG> with commit SHA = {docker_args.commit_sha}")
+    replace(k8s_config, IMAGE_TAG_TO_REPLACE, docker_args.commit_sha)
+
+
+def _replace_docker_repository_url(
+    k8s_config: pathlib.Path, docker_args: DockerArgs
+) -> None:
+    echo_info(
+        "Replacing <DOCKER_REPOSITORY_URL> with repository URL = "
+        f"{docker_args.repository}"
+    )
+    replace(k8s_config, DOCKER_REPOSITORY_URL_TO_REPLACE, docker_args.repository)
+
+
+def _docker_build(docker_args: DockerArgs):
+    try:
+        import docker
+    except ModuleNotFoundError:
+        echo_error(
+            "'docker' not installed. Run 'pip install data-pipelines-cli[docker]'"
+        )
+        sys.exit(1)
+
+    echo_info("Building Docker image")
+    docker_client = docker.from_env()
+    docker_tag = docker_args.docker_tag()
+    _, logs_generator = docker_client.images.build(path=".", tag=docker_tag)
+    click.echo(
+        "".join(
+            map(
+                lambda log: log["stream"],
+                filter(lambda log: "stream" in log, logs_generator),
+            )
+        )
+    )
+
+
+def _dbt_compile(env: str) -> None:
+    echo_info("Running dbt commands:")
+
+    echo_subinfo("dbt compile")
+    dbt(("compile",), env)
+
+    echo_subinfo("dbt docs generate")
+    dbt(("docs", "generate"), env)
+
+    echo_subinfo("dbt source freshness")
+    dbt(("source", "freshness"), env)
+
+
+def _copy_dbt_manifest() -> None:
+    echo_info("Copying DBT manifest")
+    shutil.copyfile(
+        pathlib.Path.cwd().joinpath("target", "manifest.json"),
+        BUILD_DIR.joinpath("dag", "manifest.json"),
+    )
+
+
+def _replace_datahub_address(datahub_address: str) -> None:
+    echo_info(f"Replacing INGEST_ENDPOINT with datahub address = {datahub_address}")
+    replace(
+        BUILD_DIR.joinpath("dag", "config", "base", "datahub.yml"),
+        INGEST_ENDPOINT_TO_REPLACE,
+        datahub_address,
+    )
+
+
+def _copy_src_dir_to_dst_dir(src_dir: pathlib.Path, dst_dir: pathlib.Path) -> None:
+    # It has to be deleted before copying, as `copytree` complains with
+    # `FileExistsError`
+    if pathlib.Path.exists(dst_dir):
+        shutil.rmtree(dst_dir)
+    shutil.copytree(src_dir, dst_dir)
+
+
+def _copy_dag_dir_to_build_dir() -> None:
+    dag_src_path = pathlib.Path.cwd().joinpath("dag")
+    dag_dst_path = BUILD_DIR.joinpath("dag")
+    _copy_src_dir_to_dst_dir(dag_src_path, dag_dst_path)
+
+
+def _copy_config_dir_to_build_dir() -> None:
+    config_src_path = pathlib.Path.cwd().joinpath("config")
+    dag_dst_path = BUILD_DIR.joinpath("dag", "config")
+    echo_info(f"Copying 'config' directory to {dag_dst_path}")
+    _copy_src_dir_to_dst_dir(config_src_path, dag_dst_path)
+
+
+def compile_project(
+    repository: Optional[str], datahub: Optional[str], docker_build: bool, env: str
+) -> None:
+    datahub_address = get_argument_or_environment_variable(
+        datahub, "datahub", DATAHUB_URL_ENV
+    )
+    docker_args = DockerArgs(repository)
+
+    _copy_dag_dir_to_build_dir()
+    _copy_config_dir_to_build_dir()
+
+    k8s_config: pathlib.Path = BUILD_DIR.joinpath("dag", "config", "base", "k8s.yml")
+    _replace_image_tag(k8s_config, docker_args)
+    _replace_docker_repository_url(k8s_config, docker_args)
+
+    if docker_build:
+        _docker_build(docker_args)
+    _dbt_compile(env)
+    _copy_dbt_manifest()
+
+    _replace_datahub_address(datahub_address)
+
+
+@click.command(name="compile")
+@click.option("--repository", default=None)
+@click.option("--datahub", default=None)
+@click.option("--docker-build", is_flag=True, default=False)
+@click.option("--env", default="gitlab")
+def compile_project_command(
+    repository: Optional[str], datahub: Optional[str], docker_build: bool, env: str
+) -> None:
+    compile_project(repository, datahub, docker_build, env)
diff --git a/data_pipelines_cli/cli_commands/create.py b/data_pipelines_cli/cli_commands/create.py
new file mode 100644
index 0000000..f49c133
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/create.py
@@ -0,0 +1,32 @@
+import sys
+
+import click
+import copier
+import questionary
+
+from data_pipelines_cli.cli_utils import echo_error
+from data_pipelines_cli.data_structures import read_config_or_exit
+
+
+def create(project_path: str) -> None:
+    config = read_config_or_exit()
+    if len(config["templates"]) == 0:
+        echo_error(
+            "No template provided. Run 'dp template new' to define one.",
+        )
+        sys.exit(1)
+
+    template_name = questionary.select(
+        "", choices=list(config["templates"].keys())
+    ).ask()
+    template_config = config["templates"][template_name]
+    copier.copy(src_path=template_config["template_path"], dst_path=project_path)
+
+
+@click.command(name="create")
+@click.argument(
+    "project_path",
+    type=click.Path(writable=True, path_type=str, dir_okay=True, file_okay=False),
+)
+def create_command(project_path: str) -> None:
+    create(project_path)
diff --git a/data_pipelines_cli/cli_commands/dbt.py b/data_pipelines_cli/cli_commands/dbt.py
new file mode 100644
index 0000000..74c835d
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/dbt.py
@@ -0,0 +1,15 @@
+import subprocess
+from typing import Tuple
+
+import click
+
+
+def dbt(command: Tuple[str, ...], target: str) -> None:
+    subprocess.run(["dbt", *command, "--profiles-dir", ".", "--target", target])
+
+
+@click.command(name="dbt")
+@click.argument("command", nargs=-1)
+@click.option("-t", "--target", default="local", type=str)
+def dbt_command(command: Tuple[str, ...], target: str) -> None:
+    dbt(command, target)
diff --git a/data_pipelines_cli/cli_commands/deploy.py b/data_pipelines_cli/cli_commands/deploy.py
new file mode 100644
index 0000000..ac7abba
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/deploy.py
@@ -0,0 +1,104 @@
+import subprocess
+import sys
+from typing import Optional
+
+import click
+
+from ..cli_constants import BUILD_DIR, GS_BUCKET_ENV
+from ..cli_utils import (
+    echo_error,
+    echo_info,
+    echo_subinfo,
+    get_argument_or_environment_variable,
+)
+from ..data_structures import DockerArgs
+
+
+def _docker_push(docker_args: DockerArgs):
+    try:
+        import docker
+    except ModuleNotFoundError:
+        echo_error(
+            "'docker' not installed. Run 'pip install data-pipelines-cli[docker]'"
+        )
+        sys.exit(1)
+
+    echo_info("Pushing Docker image")
+    docker_client = docker.from_env()
+    for line in docker_client.images.push(
+        repository=docker_args.repository,
+        tag=docker_args.docker_tag(),
+        stream=True,
+    ):
+        echo_subinfo(line)
+
+
+def _datahub_ingest():
+    try:
+        import datahub  # noqa: F401
+    except ModuleNotFoundError:
+        echo_error(
+            "'datahub' not installed. Run 'pip install data-pipelines-cli[datahub]'"
+        )
+        sys.exit(1)
+
+    echo_info("Ingesting datahub metadata")
+    subprocess.run(
+        [
+            "datahub",
+            "ingest",
+            "-c",
+            str(BUILD_DIR.joinpath("dag", "config", "base", "datahub.yml")),
+        ]
+    )
+    return True
+
+
+def _sync_bucket(gs_bucket: str) -> None:
+    build_dag_path = BUILD_DIR.joinpath("dag")
+
+    echo_info("Syncing GCP Bucket")
+    subprocess.run(
+        [
+            "gsutil",
+            "rsync",
+            "-d",
+            "-r",
+            str(build_dag_path),
+            f"{gs_bucket}/dags/pipeline-example",
+        ]
+    )
+
+
+def deploy(
+    repository: Optional[str],
+    gs_bucket: Optional[str],
+    docker_push: bool,
+    datahub_ingest: bool,
+) -> None:
+    gs_bucket = get_argument_or_environment_variable(
+        gs_bucket, "gs-bucket", GS_BUCKET_ENV
+    )
+
+    if docker_push:
+        docker_args = DockerArgs(repository)
+        _docker_push(docker_args)
+
+    if datahub_ingest:
+        _datahub_ingest()
+
+    _sync_bucket(gs_bucket)
+
+
+@click.command(name="deploy")
+@click.option("--repository", default=None)
+@click.option("--gs-bucket", default=None)
+@click.option("--docker-push", is_flag=True, default=False)
+@click.option("--datahub-ingest", is_flag=True, default=False)
+def deploy_command(
+    repository: Optional[str],
+    gs_bucket: Optional[str],
+    docker_push: bool,
+    datahub_ingest: bool,
+) -> None:
+    deploy(repository, gs_bucket, docker_push, datahub_ingest)
diff --git a/data_pipelines_cli/cli_commands/init.py b/data_pipelines_cli/cli_commands/init.py
new file mode 100644
index 0000000..8bbc824
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/init.py
@@ -0,0 +1,37 @@
+import getpass
+import sys
+
+import click
+import questionary
+import yaml
+
+from data_pipelines_cli.cli_constants import CONFIGURATION_PATH
+from data_pipelines_cli.data_structures import DataPipelinesConfig
+
+
+def init() -> None:
+    if CONFIGURATION_PATH.is_file():
+        overwrite_confirm = questionary.confirm(
+            "dp config already exists. Do you want to overwrite it?",
+            default=False,
+        ).ask()
+        if not overwrite_confirm:
+            sys.exit(1)
+
+    username: str = questionary.text(
+        "What's your username?",
+        default=getpass.getuser(),
+        validate=lambda text: len(text) > 0,
+    ).ask()
+
+    config: DataPipelinesConfig = {
+        "username": username,
+        "templates": {},
+    }
+    with open(CONFIGURATION_PATH, "w") as f:
+        yaml.dump(config, f, default_flow_style=False)
+
+
+@click.command(name="init")
+def init_command() -> None:
+    init()
diff --git a/data_pipelines_cli/cli_commands/template.py b/data_pipelines_cli/cli_commands/template.py
new file mode 100644
index 0000000..09a0501
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/template.py
@@ -0,0 +1,58 @@
+import sys
+
+import click
+import questionary
+import yaml
+
+from data_pipelines_cli.cli_constants import CONFIGURATION_PATH
+from data_pipelines_cli.data_structures import read_config_or_exit
+
+
+def new_template() -> None:
+    config = read_config_or_exit()
+    template_name = questionary.text(
+        "Name of your project template", validate=lambda text: len(text) > 0
+    ).ask()
+    if config["templates"].get(template_name):
+        overwrite_confirm = questionary.confirm(
+            f"Project template named {template_name} already exists. "
+            f"Do you want to overwrite it?",
+            default=False,
+        ).ask()
+        if not overwrite_confirm:
+            sys.exit(1)
+    template_path: str = questionary.text(
+        "Template path (repository address)",
+        validate=lambda text: len(text) > 0,
+    ).ask()
+
+    config["templates"][template_name] = {
+        "template_name": template_name,
+        "template_path": template_path,
+    }
+
+    with open(CONFIGURATION_PATH, "w") as f:
+        yaml.dump(config, f, default_flow_style=False)
+
+
+def list_templates() -> None:
+    config = read_config_or_exit()
+
+    click.echo("AVAILABLE TEMPLATES:\n")
+    for tc in config["templates"].values():
+        click.echo(yaml.dump(tc))
+
+
+@click.group(name="template")
+def template_group() -> None:
+    pass
+
+
+@template_group.command(name="new")
+def new_template_command() -> None:
+    new_template()
+
+
+@template_group.command(name="list")
+def list_templates_command() -> None:
+    list_templates()
diff --git a/data_pipelines_cli/cli_constants.py b/data_pipelines_cli/cli_constants.py
new file mode 100644
index 0000000..4434318
--- /dev/null
+++ b/data_pipelines_cli/cli_constants.py
@@ -0,0 +1,9 @@
+import pathlib
+
+CONFIGURATION_PATH: pathlib.Path = pathlib.Path.home().joinpath(".dp.yml")
+DATAHUB_URL_ENV: str = "DATAHUB_URL"
+GS_BUCKET_ENV: str = "GS_BUCKET"
+IMAGE_TAG_TO_REPLACE: str = "<IMAGE_TAG>"
+DOCKER_REPOSITORY_URL_TO_REPLACE: str = "<DOCKER_REPOSITORY_URL>"
+INGEST_ENDPOINT_TO_REPLACE: str = "<INGEST_ENDPOINT>"
+BUILD_DIR: pathlib.Path = pathlib.Path.cwd().joinpath("build")
diff --git a/data_pipelines_cli/cli_utils.py b/data_pipelines_cli/cli_utils.py
new file mode 100644
index 0000000..a0475e3
--- /dev/null
+++ b/data_pipelines_cli/cli_utils.py
@@ -0,0 +1,31 @@
+import os
+import sys
+from typing import Optional
+
+import click
+
+
+def echo_error(text: str, **kwargs):
+    click.secho(text, file=sys.stderr, fg="red", **kwargs)
+
+
+def echo_info(text: str, **kwargs):
+    click.secho(text, fg="blue", bold=True, **kwargs)
+
+
+def echo_subinfo(text: str, **kwargs):
+    click.secho(text, fg="bright_blue", **kwargs)
+
+
+def get_argument_or_environment_variable(
+    argument: Optional[str], argument_name: str, environment_variable_name: str
+) -> str:
+    result = argument or os.environ.get(environment_variable_name)
+    if not result:
+        echo_error(
+            f"Could not get {environment_variable_name}. Either set it as an "
+            f"environment variable {environment_variable_name} or pass as a "
+            f"`--{argument_name}` CLI argument"
+        )
+        sys.exit(1)
+    return result
diff --git a/data_pipelines_cli/data_structures.py b/data_pipelines_cli/data_structures.py
new file mode 100644
index 0000000..f536ca2
--- /dev/null
+++ b/data_pipelines_cli/data_structures.py
@@ -0,0 +1,55 @@
+import sys
+from typing import Dict, Optional
+
+import yaml
+
+from data_pipelines_cli.cli_constants import CONFIGURATION_PATH
+from data_pipelines_cli.cli_utils import (
+    echo_error,
+    get_argument_or_environment_variable,
+)
+from data_pipelines_cli.io_utils import git_revision_hash
+
+if sys.version_info >= (3, 8):
+    from typing import TypedDict  # pylint: disable=no-name-in-module
+else:
+    from typing_extensions import TypedDict
+
+
+class TemplateConfig(TypedDict):
+    template_name: str
+    template_path: str
+
+
+class DataPipelinesConfig(TypedDict):
+    username: str
+    templates: Dict[str, TemplateConfig]
+
+
+def read_config_or_exit() -> DataPipelinesConfig:
+    if not CONFIGURATION_PATH.is_file():
+        echo_error(
+            "No configuration file found. Run 'dp init' to create it.",
+        )
+        sys.exit(1)
+
+    with open(CONFIGURATION_PATH, "r") as f:
+        return yaml.safe_load(f)
+
+
+class DockerArgs:
+    repository: str
+    commit_sha: str
+
+    def __init__(self, repository: Optional[str]):
+        self.repository = get_argument_or_environment_variable(
+            repository, "repository", "REPOSITORY_URL"
+        )
+        commit_sha = git_revision_hash()
+        if not commit_sha:
+            echo_error("Could not get git revision hash.")
+            sys.exit(1)
+        self.commit_sha = commit_sha
+
+    def docker_tag(self) -> str:
+        return f"{self.repository}:{self.commit_sha}"
diff --git a/data_pipelines_cli/io_utils.py b/data_pipelines_cli/io_utils.py
new file mode 100644
index 0000000..e0bd63d
--- /dev/null
+++ b/data_pipelines_cli/io_utils.py
@@ -0,0 +1,59 @@
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Optional
+
+import click
+
+
+# Python's `sed` equivalent, based on the following answer:
+# https://stackoverflow.com/a/31499114
+def replace(filename, pattern, replacement):
+    """
+    Perform the pure-Python equivalent of in-place `sed` substitution: e.g.,
+    `sed -i -e 's/'${pattern}'/'${replacement}' "${filename}"`.
+
+    Beware however, it uses Python regex dialect instead of `sed`'s one.
+    It can introduce regex-related bugs.
+    """
+    # For efficiency, precompile the passed regular expression.
+    pattern_compiled = re.compile(pattern)
+
+    # For portability, NamedTemporaryFile() defaults to mode "w+b" (i.e.,
+    # binary writing with updating). This is usually a good thing. In this
+    # case, however, binary writing imposes non-trivial encoding constraints
+    # trivially resolved by switching to text writing. Let's do that.
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
+        with open(filename) as src_file:
+            for line in src_file:
+                tmp_file.write(pattern_compiled.sub(replacement, line))
+
+    # Overwrite the original file with the munged temporary file in a
+    # manner preserving file attributes (e.g., permissions).
+    shutil.copystat(filename, tmp_file.name)
+    shutil.move(tmp_file.name, filename)
+
+
+def git_revision_hash() -> Optional[str]:
+    try:
+        rev_process = subprocess.run(
+            ["git", "rev-parse", "HEAD"], check=True, capture_output=True
+        )
+        return rev_process.stdout.decode("ascii").strip()
+    except FileNotFoundError:
+        click.echo(
+            "The tool has run across an error when trying to get Git "
+            "revision hash. Ensure you have `git` installed",
+            file=sys.stderr,
+        )
+        return None
+    except subprocess.CalledProcessError as err:
+        click.echo(
+            "The tool has run across a following error when trying to "
+            "get Git revision hash:",
+            file=sys.stderr,
+        )
+        click.echo(err.stderr, file=sys.stderr)
+        return None
diff --git a/pyproject.toml b/pyproject.toml
index a8f43fe..e69de29 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,2 +0,0 @@
-[tool.black]
-line-length = 79
diff --git a/setup.py b/setup.py
index 9ddc641..fa152eb 100644
--- a/setup.py
+++ b/setup.py
@@ -11,15 +11,18 @@
     "pyyaml>=5.1, <6.0",
     "types-PyYAML>=6.0",
     "copier==5.1.0",
+    "dbt>=0.21, <0.22",
 ]
 
 EXTRA_REQUIRE = {
+    "docker": ["docker>=5.0"],
+    "datahub": ["acryl-datahub>=0.8.17, <0.8.18"],
     "tests": [
         "pytest>=6.2.2, <7.0.0",
         "pytest-cov>=2.8.0, <3.0.0",
         "pre-commit==2.15.0",
         "tox==3.21.1",
-    ]
+    ],
 }
 
 setup(


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 4ab3bd3..8cb466f 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -4,8 +4,12 @@
 
 from click.testing import CliRunner
 
-import data_pipelines_cli.cli as cli
-from data_pipelines_cli.cli import DataPipelinesConfig, TemplateConfig
+from data_pipelines_cli.cli_commands.template import list_templates_command
+from data_pipelines_cli.data_structures import (
+    DataPipelinesConfig,
+    TemplateConfig,
+    read_config_or_exit,
+)
 
 
 class MyTestCase(unittest.TestCase):
@@ -22,26 +26,22 @@ class MyTestCase(unittest.TestCase):
             ),
         },
     )
-    example_config_path = pathlib.Path(__file__).parent.joinpath(
-        "example_config.yml"
-    )
+    example_config_path = pathlib.Path(__file__).parent.joinpath("example_config.yml")
 
     def test_read_config(self):
         with patch(
-            "data_pipelines_cli.cli.CONFIGURATION_PATH",
+            "data_pipelines_cli.data_structures.CONFIGURATION_PATH",
             self.example_config_path,
         ):
-            self.assertEqual(
-                self.example_config_dict, cli.read_config_or_exit()
-            )
+            self.assertEqual(self.example_config_dict, read_config_or_exit())
 
     def test_list_templates(self):
         runner = CliRunner()
         with patch(
-            "data_pipelines_cli.cli.CONFIGURATION_PATH",
+            "data_pipelines_cli.data_structures.CONFIGURATION_PATH",
             self.example_config_path,
         ):
-            result = runner.invoke(cli.list_templates)
+            result = runner.invoke(list_templates_command)
             self.assertEqual(0, result.exit_code)
             self.assertEqual(
                 "AVAILABLE TEMPLATES:\n\n"