Repository: getindata/data-pipelines-cli (language: python)

Issue:
Fix `isort` clashing with `black`
According to [this guide](https://github.com/psf/black/blob/main/docs/guides/using_black_with_other_tools.md), `black` tends to clash with both `isort` and `Flake8` when it comes to formatting. This commit fixes the problem by following the guide in the mentioned article, i.e. by setting `isort` configuration to be `black`-compatible and ignoring a whitespace warning in `Flake8`.

---
Keep in mind:
- [ ] Documentation updates
- [ ] [Changelog](CHANGELOG.md) updates
Release 0.1.0
Bump version and CHANGELOG for next release.


Discussion / hints:

Fix patch:
diff --git a/.isort.cfg b/.isort.cfg
index 11eecb6..3d5e5bc 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -1,9 +1,5 @@
-[tool:isort]
-include_trailing_comma = True
-multi_line_output = 3
-line_length = 79
+[settings]
+profile = black
 known_first_party = data_pipelines_cli
 default_section = THIRDPARTY
-
-[settings]
 known_third_party = setuptools
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 369b32e..f5ef28d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,12 +1,12 @@
 repos:
 
 - repo: https://github.com/pre-commit/mirrors-isort
-  rev: v4.3.21
+  rev: v5.9.3
   hooks:
     - id: isort
 
 - repo: https://github.com/psf/black
-  rev: stable
+  rev: 21.11b1
   hooks:
     - id: black
 
@@ -25,6 +25,8 @@ repos:
       ]
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.910
+  rev: v0.910-1
   hooks:
     - id: mypy
+      additional_dependencies:
+        - 'types-PyYAML'
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 825c32f..3663d68 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,1 +1,14 @@
 # Changelog
+
+## [Unreleased]
+
+## [0.1.0] - 2021-12-01
+
+### Added
+
+-   Draft of `dp init`, `dp create`, `dp template new`, `dp template list` and `dp dbt`
+-   Draft of `dp compile` and `dp deploy`
+
+[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.1.0...HEAD
+
+[0.1.0]: https://github.com/getindata/data-pipelines-cli/compare/5df87160e660d372a52a3665fe79be2029089613...0.1.0
diff --git a/README.md b/README.md
index 4ced80c..6fdce97 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,16 @@
 # data-pipelines-cli
+
 CLI for data platform
+
+## Installation
+TODO
+
+## Usage
+First, run `dp init` to configure your first template. `dp init` will allow you to add more templates later. Then, you can use `dp create <new-project-path>` to choose one of the templates added before and create the project in `<new-project-path>` directory.
+
+`dp list` lists all added templates.
+
+## Contributing
+Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change.
+
+Please make sure to update tests as appropriate.
\ No newline at end of file
diff --git a/data_pipelines_cli/__init__.py b/data_pipelines_cli/__init__.py
index d56d88f..9f7a875 100644
--- a/data_pipelines_cli/__init__.py
+++ b/data_pipelines_cli/__init__.py
@@ -1,1 +1,1 @@
-version = "0.0.0"
+version = "0.1.0"
diff --git a/data_pipelines_cli/cli.py b/data_pipelines_cli/cli.py
index ed5e6ff..db1085e 100644
--- a/data_pipelines_cli/cli.py
+++ b/data_pipelines_cli/cli.py
@@ -1,15 +1,21 @@
 import click
 
+from .cli_commands.compile import compile_project_command
+from .cli_commands.create import create_command
+from .cli_commands.dbt import dbt_command
+from .cli_commands.deploy import deploy_command
+from .cli_commands.init import init_command
+from .cli_commands.template import template_group
+
 
 @click.group()
 def cli() -> None:
     pass
 
 
-@cli.command()
-def hello() -> None:
-    print("Hello world!")
-
-
-if __name__ == "__main__":
-    cli()
+cli.add_command(init_command)
+cli.add_command(template_group)
+cli.add_command(create_command)
+cli.add_command(dbt_command)
+cli.add_command(compile_project_command)
+cli.add_command(deploy_command)
diff --git a/data_pipelines_cli/cli_commands/__init__.py b/data_pipelines_cli/cli_commands/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/data_pipelines_cli/cli_commands/compile.py b/data_pipelines_cli/cli_commands/compile.py
new file mode 100644
index 0000000..ec72dd9
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/compile.py
@@ -0,0 +1,146 @@
+import pathlib
+import shutil
+import sys
+from typing import Optional
+
+import click
+
+from ..cli_constants import (
+    BUILD_DIR,
+    DATAHUB_URL_ENV,
+    DOCKER_REPOSITORY_URL_TO_REPLACE,
+    IMAGE_TAG_TO_REPLACE,
+    INGEST_ENDPOINT_TO_REPLACE,
+)
+from ..cli_utils import (
+    echo_error,
+    echo_info,
+    echo_subinfo,
+    get_argument_or_environment_variable,
+)
+from ..data_structures import DockerArgs
+from ..io_utils import replace
+from .dbt import dbt
+
+
+def _replace_image_tag(k8s_config: pathlib.Path, docker_args: DockerArgs) -> None:
+    echo_info(f"Replacing <IMAGE_TAG> with commit SHA = {docker_args.commit_sha}")
+    replace(k8s_config, IMAGE_TAG_TO_REPLACE, docker_args.commit_sha)
+
+
+def _replace_docker_repository_url(
+    k8s_config: pathlib.Path, docker_args: DockerArgs
+) -> None:
+    echo_info(
+        "Replacing <DOCKER_REPOSITORY_URL> with repository URL = "
+        f"{docker_args.repository}"
+    )
+    replace(k8s_config, DOCKER_REPOSITORY_URL_TO_REPLACE, docker_args.repository)
+
+
+def _docker_build(docker_args: DockerArgs):
+    try:
+        import docker
+    except ModuleNotFoundError:
+        echo_error(
+            "'docker' not installed. Run 'pip install data-pipelines-cli[docker]'"
+        )
+        sys.exit(1)
+
+    echo_info("Building Docker image")
+    docker_client = docker.from_env()
+    docker_tag = docker_args.docker_tag()
+    _, logs_generator = docker_client.images.build(path=".", tag=docker_tag)
+    click.echo(
+        "".join(
+            map(
+                lambda log: log["stream"],
+                filter(lambda log: "stream" in log, logs_generator),
+            )
+        )
+    )
+
+
+def _dbt_compile(env: str) -> None:
+    echo_info("Running dbt commands:")
+
+    echo_subinfo("dbt compile")
+    dbt(("compile",), env)
+
+    echo_subinfo("dbt docs generate")
+    dbt(("docs", "generate"), env)
+
+    echo_subinfo("dbt source freshness")
+    dbt(("source", "freshness"), env)
+
+
+def _copy_dbt_manifest() -> None:
+    echo_info("Copying DBT manifest")
+    shutil.copyfile(
+        pathlib.Path.cwd().joinpath("target", "manifest.json"),
+        BUILD_DIR.joinpath("dag", "manifest.json"),
+    )
+
+
+def _replace_datahub_address(datahub_address: str) -> None:
+    echo_info(f"Replacing INGEST_ENDPOINT with datahub address = {datahub_address}")
+    replace(
+        BUILD_DIR.joinpath("dag", "config", "base", "datahub.yml"),
+        INGEST_ENDPOINT_TO_REPLACE,
+        datahub_address,
+    )
+
+
+def _copy_src_dir_to_dst_dir(src_dir: pathlib.Path, dst_dir: pathlib.Path) -> None:
+    # It has to be deleted before copying, as `copytree` complains with
+    # `FileExistsError`
+    if pathlib.Path.exists(dst_dir):
+        shutil.rmtree(dst_dir)
+    shutil.copytree(src_dir, dst_dir)
+
+
+def _copy_dag_dir_to_build_dir() -> None:
+    dag_src_path = pathlib.Path.cwd().joinpath("dag")
+    dag_dst_path = BUILD_DIR.joinpath("dag")
+    _copy_src_dir_to_dst_dir(dag_src_path, dag_dst_path)
+
+
+def _copy_config_dir_to_build_dir() -> None:
+    config_src_path = pathlib.Path.cwd().joinpath("config")
+    dag_dst_path = BUILD_DIR.joinpath("dag", "config")
+    echo_info(f"Copying 'config' directory to {dag_dst_path}")
+    _copy_src_dir_to_dst_dir(config_src_path, dag_dst_path)
+
+
+def compile_project(
+    repository: Optional[str], datahub: Optional[str], docker_build: bool, env: str
+) -> None:
+    datahub_address = get_argument_or_environment_variable(
+        datahub, "datahub", DATAHUB_URL_ENV
+    )
+    docker_args = DockerArgs(repository)
+
+    _copy_dag_dir_to_build_dir()
+    _copy_config_dir_to_build_dir()
+
+    k8s_config: pathlib.Path = BUILD_DIR.joinpath("dag", "config", "base", "k8s.yml")
+    _replace_image_tag(k8s_config, docker_args)
+    _replace_docker_repository_url(k8s_config, docker_args)
+
+    if docker_build:
+        _docker_build(docker_args)
+    _dbt_compile(env)
+    _copy_dbt_manifest()
+
+    _replace_datahub_address(datahub_address)
+
+
+@click.command(name="compile")
+@click.option("--repository", default=None)
+@click.option("--datahub", default=None)
+@click.option("--docker-build", is_flag=True, default=False)
+@click.option("--env", default="gitlab")
+def compile_project_command(
+    repository: Optional[str], datahub: Optional[str], docker_build: bool, env: str
+) -> None:
+    compile_project(repository, datahub, docker_build, env)
diff --git a/data_pipelines_cli/cli_commands/create.py b/data_pipelines_cli/cli_commands/create.py
new file mode 100644
index 0000000..f49c133
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/create.py
@@ -0,0 +1,32 @@
+import sys
+
+import click
+import copier
+import questionary
+
+from data_pipelines_cli.cli_utils import echo_error
+from data_pipelines_cli.data_structures import read_config_or_exit
+
+
+def create(project_path: str) -> None:
+    config = read_config_or_exit()
+    if len(config["templates"]) == 0:
+        echo_error(
+            "No template provided. Run 'dp template new' to define one.",
+        )
+        sys.exit(1)
+
+    template_name = questionary.select(
+        "", choices=list(config["templates"].keys())
+    ).ask()
+    template_config = config["templates"][template_name]
+    copier.copy(src_path=template_config["template_path"], dst_path=project_path)
+
+
+@click.command(name="create")
+@click.argument(
+    "project_path",
+    type=click.Path(writable=True, path_type=str, dir_okay=True, file_okay=False),
+)
+def create_command(project_path: str) -> None:
+    create(project_path)
diff --git a/data_pipelines_cli/cli_commands/dbt.py b/data_pipelines_cli/cli_commands/dbt.py
new file mode 100644
index 0000000..74c835d
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/dbt.py
@@ -0,0 +1,15 @@
+import subprocess
+from typing import Tuple
+
+import click
+
+
+def dbt(command: Tuple[str, ...], target: str) -> None:
+    subprocess.run(["dbt", *command, "--profiles-dir", ".", "--target", target])
+
+
+@click.command(name="dbt")
+@click.argument("command", nargs=-1)
+@click.option("-t", "--target", default="local", type=str)
+def dbt_command(command: Tuple[str, ...], target: str) -> None:
+    dbt(command, target)
diff --git a/data_pipelines_cli/cli_commands/deploy.py b/data_pipelines_cli/cli_commands/deploy.py
new file mode 100644
index 0000000..ac7abba
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/deploy.py
@@ -0,0 +1,104 @@
+import subprocess
+import sys
+from typing import Optional
+
+import click
+
+from ..cli_constants import BUILD_DIR, GS_BUCKET_ENV
+from ..cli_utils import (
+    echo_error,
+    echo_info,
+    echo_subinfo,
+    get_argument_or_environment_variable,
+)
+from ..data_structures import DockerArgs
+
+
+def _docker_push(docker_args: DockerArgs):
+    try:
+        import docker
+    except ModuleNotFoundError:
+        echo_error(
+            "'docker' not installed. Run 'pip install data-pipelines-cli[docker]'"
+        )
+        sys.exit(1)
+
+    echo_info("Pushing Docker image")
+    docker_client = docker.from_env()
+    for line in docker_client.images.push(
+        repository=docker_args.repository,
+        tag=docker_args.docker_tag(),
+        stream=True,
+    ):
+        echo_subinfo(line)
+
+
+def _datahub_ingest():
+    try:
+        import datahub  # noqa: F401
+    except ModuleNotFoundError:
+        echo_error(
+            "'datahub' not installed. Run 'pip install data-pipelines-cli[datahub]'"
+        )
+        sys.exit(1)
+
+    echo_info("Ingesting datahub metadata")
+    subprocess.run(
+        [
+            "datahub",
+            "ingest",
+            "-c",
+            str(BUILD_DIR.joinpath("dag", "config", "base", "datahub.yml")),
+        ]
+    )
+    return True
+
+
+def _sync_bucket(gs_bucket: str) -> None:
+    build_dag_path = BUILD_DIR.joinpath("dag")
+
+    echo_info("Syncing GCP Bucket")
+    subprocess.run(
+        [
+            "gsutil",
+            "rsync",
+            "-d",
+            "-r",
+            str(build_dag_path),
+            f"{gs_bucket}/dags/pipeline-example",
+        ]
+    )
+
+
+def deploy(
+    repository: Optional[str],
+    gs_bucket: Optional[str],
+    docker_push: bool,
+    datahub_ingest: bool,
+) -> None:
+    gs_bucket = get_argument_or_environment_variable(
+        gs_bucket, "gs-bucket", GS_BUCKET_ENV
+    )
+
+    if docker_push:
+        docker_args = DockerArgs(repository)
+        _docker_push(docker_args)
+
+    if datahub_ingest:
+        _datahub_ingest()
+
+    _sync_bucket(gs_bucket)
+
+
+@click.command(name="deploy")
+@click.option("--repository", default=None)
+@click.option("--gs-bucket", default=None)
+@click.option("--docker-push", is_flag=True, default=False)
+@click.option("--datahub-ingest", is_flag=True, default=False)
+def deploy_command(
+    repository: Optional[str],
+    gs_bucket: Optional[str],
+    docker_push: bool,
+    datahub_ingest: bool,
+) -> None:
+    deploy(repository, gs_bucket, docker_push, datahub_ingest)
diff --git a/data_pipelines_cli/cli_commands/init.py b/data_pipelines_cli/cli_commands/init.py
new file mode 100644
index 0000000..8bbc824
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/init.py
@@ -0,0 +1,37 @@
+import getpass
+import sys
+
+import click
+import questionary
+import yaml
+
+from data_pipelines_cli.cli_constants import CONFIGURATION_PATH
+from data_pipelines_cli.data_structures import DataPipelinesConfig
+
+
+def init() -> None:
+    if CONFIGURATION_PATH.is_file():
+        overwrite_confirm = questionary.confirm(
+            "dp config already exists. Do you want to overwrite it?",
+            default=False,
+        ).ask()
+        if not overwrite_confirm:
+            sys.exit(1)
+
+    username: str = questionary.text(
+        "What's your username?",
+        default=getpass.getuser(),
+        validate=lambda text: len(text) > 0,
+    ).ask()
+
+    config: DataPipelinesConfig = {
+        "username": username,
+        "templates": {},
+    }
+    with open(CONFIGURATION_PATH, "w") as f:
+        yaml.dump(config, f, default_flow_style=False)
+
+
+@click.command(name="init")
+def init_command() -> None:
+    init()
diff --git a/data_pipelines_cli/cli_commands/template.py b/data_pipelines_cli/cli_commands/template.py
new file mode 100644
index 0000000..09a0501
--- /dev/null
+++ b/data_pipelines_cli/cli_commands/template.py
@@ -0,0 +1,58 @@
+import sys
+
+import click
+import questionary
+import yaml
+
+from data_pipelines_cli.cli_constants import CONFIGURATION_PATH
+from data_pipelines_cli.data_structures import read_config_or_exit
+
+
+def new_template() -> None:
+    config = read_config_or_exit()
+    template_name = questionary.text(
+        "Name of your project template", validate=lambda text: len(text) > 0
+    ).ask()
+    if config["templates"].get(template_name):
+        overwrite_confirm = questionary.confirm(
+            f"Project template named {template_name} already exists. "
+            f"Do you want to overwrite it?",
+            default=False,
+        ).ask()
+        if not overwrite_confirm:
+            sys.exit(1)
+    template_path: str = questionary.text(
+        "Template path (repository address)",
+        validate=lambda text: len(text) > 0,
+    ).ask()
+
+    config["templates"][template_name] = {
+        "template_name": template_name,
+        "template_path": template_path,
+    }
+
+    with open(CONFIGURATION_PATH, "w") as f:
+        yaml.dump(config, f, default_flow_style=False)
+
+
+def list_templates() -> None:
+    config = read_config_or_exit()
+
+    click.echo("AVAILABLE TEMPLATES:\n")
+    for tc in config["templates"].values():
+        click.echo(yaml.dump(tc))
+
+
+@click.group(name="template")
+def template_group() -> None:
+    pass
+
+
+@template_group.command(name="new")
+def new_template_command() -> None:
+    new_template()
+
+
+@template_group.command(name="list")
+def list_templates_command() -> None:
+    list_templates()
diff --git a/data_pipelines_cli/cli_constants.py b/data_pipelines_cli/cli_constants.py
new file mode 100644
index 0000000..4434318
--- /dev/null
+++ b/data_pipelines_cli/cli_constants.py
@@ -0,0 +1,9 @@
+import pathlib
+
+CONFIGURATION_PATH: pathlib.Path = pathlib.Path.home().joinpath(".dp.yml")
+DATAHUB_URL_ENV: str = "DATAHUB_URL"
+GS_BUCKET_ENV: str = "GS_BUCKET"
+IMAGE_TAG_TO_REPLACE: str = "<IMAGE_TAG>"
+DOCKER_REPOSITORY_URL_TO_REPLACE: str = "<DOCKER_REPOSITORY_URL>"
+INGEST_ENDPOINT_TO_REPLACE: str = "<INGEST_ENDPOINT>"
+BUILD_DIR: pathlib.Path = pathlib.Path.cwd().joinpath("build")
diff --git a/data_pipelines_cli/cli_utils.py b/data_pipelines_cli/cli_utils.py
new file mode 100644
index 0000000..a0475e3
--- /dev/null
+++ b/data_pipelines_cli/cli_utils.py
@@ -0,0 +1,31 @@
+import os
+import sys
+from typing import Optional
+
+import click
+
+
+def echo_error(text: str, **kwargs):
+    click.secho(text, file=sys.stderr, fg="red", **kwargs)
+
+
+def echo_info(text: str, **kwargs):
+    click.secho(text, fg="blue", bold=True, **kwargs)
+
+
+def echo_subinfo(text: str, **kwargs):
+    click.secho(text, fg="bright_blue", **kwargs)
+
+
+def get_argument_or_environment_variable(
+    argument: Optional[str], argument_name: str, environment_variable_name: str
+) -> str:
+    result = argument or os.environ.get(environment_variable_name)
+    if not result:
+        echo_error(
+            f"Could not get {environment_variable_name}. Either set it as an "
+            f"environment variable {environment_variable_name} or pass as a "
+            f"`--{argument_name}` CLI argument"
+        )
+        sys.exit(1)
+    return result
diff --git a/data_pipelines_cli/data_structures.py b/data_pipelines_cli/data_structures.py
new file mode 100644
index 0000000..f536ca2
--- /dev/null
+++ b/data_pipelines_cli/data_structures.py
@@ -0,0 +1,55 @@
+import sys
+from typing import Dict, Optional
+
+import yaml
+
+from data_pipelines_cli.cli_constants import CONFIGURATION_PATH
+from data_pipelines_cli.cli_utils import (
+    echo_error,
+    get_argument_or_environment_variable,
+)
+from data_pipelines_cli.io_utils import git_revision_hash
+
+if sys.version_info >= (3, 8):
+    from typing import TypedDict  # pylint: disable=no-name-in-module
+else:
+    from typing_extensions import TypedDict
+
+
+class TemplateConfig(TypedDict):
+    template_name: str
+    template_path: str
+
+
+class DataPipelinesConfig(TypedDict):
+    username: str
+    templates: Dict[str, TemplateConfig]
+
+
+def read_config_or_exit() -> DataPipelinesConfig:
+    if not CONFIGURATION_PATH.is_file():
+        echo_error(
+            "No configuration file found. Run 'dp init' to create it.",
+        )
+        sys.exit(1)
+
+    with open(CONFIGURATION_PATH, "r") as f:
+        return yaml.safe_load(f)
+
+
+class DockerArgs:
+    repository: str
+    commit_sha: str
+
+    def __init__(self, repository: Optional[str]):
+        self.repository = get_argument_or_environment_variable(
+            repository, "repository", "REPOSITORY_URL"
+        )
+        commit_sha = git_revision_hash()
+        if not commit_sha:
+            echo_error("Could not get git revision hash.")
+            sys.exit(1)
+        self.commit_sha = commit_sha
+
+    def docker_tag(self) -> str:
+        return f"{self.repository}:{self.commit_sha}"
diff --git a/data_pipelines_cli/io_utils.py b/data_pipelines_cli/io_utils.py
new file mode 100644
index 0000000..e0bd63d
--- /dev/null
+++ b/data_pipelines_cli/io_utils.py
@@ -0,0 +1,59 @@
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Optional
+
+import click
+
+
+# Python's `sed` equivalent, based on the following answer:
+# https://stackoverflow.com/a/31499114
+def replace(filename, pattern, replacement):
+    """
+    Perform the pure-Python equivalent of in-place `sed` substitution: e.g.,
+    `sed -i -e 's/'${pattern}'/'${replacement}' "${filename}"`.
+
+    Beware however, it uses Python regex dialect instead of `sed`'s one.
+    It can introduce regex-related bugs.
+    """
+    # For efficiency, precompile the passed regular expression.
+    pattern_compiled = re.compile(pattern)
+
+    # For portability, NamedTemporaryFile() defaults to mode "w+b" (i.e.,
+    # binary writing with updating). This is usually a good thing. In this
+    # case, however, binary writing imposes non-trivial encoding constraints
+    # trivially resolved by switching to text writing. Let's do that.
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
+        with open(filename) as src_file:
+            for line in src_file:
+                tmp_file.write(pattern_compiled.sub(replacement, line))
+
+    # Overwrite the original file with the munged temporary file in a
+    # manner preserving file attributes (e.g., permissions).
+    shutil.copystat(filename, tmp_file.name)
+    shutil.move(tmp_file.name, filename)
+
+
+def git_revision_hash() -> Optional[str]:
+    try:
+        rev_process = subprocess.run(
+            ["git", "rev-parse", "HEAD"], check=True, capture_output=True
+        )
+        return rev_process.stdout.decode("ascii").strip()
+    except FileNotFoundError:
+        click.echo(
+            "The tool has run across an error when trying to get Git "
+            "revision hash. Ensure you have `git` installed",
+            file=sys.stderr,
+        )
+        return None
+    except subprocess.CalledProcessError as err:
+        click.echo(
+            "The tool has run across a following error when trying to "
+            "get Git revision hash:",
+            file=sys.stderr,
+        )
+        click.echo(err.stderr, file=sys.stderr)
+        return None
diff --git a/pyproject.toml b/pyproject.toml
index a8f43fe..e69de29 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,2 +0,0 @@
-[tool.black]
-line-length = 79
diff --git a/setup.cfg b/setup.cfg
index a6b4418..5f20a1e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.0
+current_version = 0.1.0
 
 [bumpversion:file:setup.py]
 
@@ -7,7 +7,8 @@ current_version = 0.0.0
 
 [flake8]
 exclude = .git,__pycache__,build,dist
-max-line-length = 79
+max-line-length = 88
+extend-ignore = E203
 
 [mypy]
 no_strict_optional = True
diff --git a/setup.py b/setup.py
index 2e9e004..9b5a35b 100644
--- a/setup.py
+++ b/setup.py
@@ -5,20 +5,29 @@
 with open("README.md") as f:
     README = f.read()
 
-INSTALL_REQUIREMENTS = ["click<8.0"]
+INSTALL_REQUIREMENTS = [
+    "click<8.0",
+    "questionary==1.10.0",
+    "pyyaml>=5.1, <6.0",
+    "types-PyYAML>=6.0",
+    "copier==5.1.0",
+    "dbt>=0.21, <0.22",
+]
 
 EXTRA_REQUIRE = {
+    "docker": ["docker>=5.0"],
+    "datahub": ["acryl-datahub>=0.8.17, <0.8.18"],
     "tests": [
         "pytest>=6.2.2, <7.0.0",
         "pytest-cov>=2.8.0, <3.0.0",
         "pre-commit==2.15.0",
         "tox==3.21.1",
-    ]
+    ],
 }
 
 setup(
     name="data_pipelines_cli",
-    version="0.0.0",
+    version="0.1.0",
     description="CLI for data platform",
     long_description=README,
     long_description_content_type="text/markdown",
diff --git a/tox.ini b/tox.ini
index ba0267e..b387cdd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,12 +6,13 @@ envlist = py37,py39
 extras =
     tests
 commands=
-    python -m pytest --cov data_pipelines_cli --cov-report xml --cov-report term-missing --ignore=venv
+    python -m pytest -s --cov data_pipelines_cli --cov-report xml --cov-report term-missing --ignore=venv
 
 # Lint
 [flake8]
 exclude = .git,__pycache__,build,dist
-max-line-length = 79
+max-line-length = 88
+extend-ignore = E203
 
 [mypy]
 no_strict_optional = True


Test patch:
diff --git a/tests/example_config.yml b/tests/example_config.yml
new file mode 100644
index 0000000..f5ff426
--- /dev/null
+++ b/tests/example_config.yml
@@ -0,0 +1,8 @@
+username: testuser
+templates:
+  template1:
+    template_name: template1
+    template_path: https://example.com/xyz/abcd.git
+  template2:
+    template_name: template2
+    template_path: https://example.com/git/example.git
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 0000000..8cb466f
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,53 @@
+import pathlib
+import unittest
+from unittest.mock import patch
+
+from click.testing import CliRunner
+
+from data_pipelines_cli.cli_commands.template import list_templates_command
+from data_pipelines_cli.data_structures import (
+    DataPipelinesConfig,
+    TemplateConfig,
+    read_config_or_exit,
+)
+
+
+class MyTestCase(unittest.TestCase):
+    example_config_dict = DataPipelinesConfig(
+        username="testuser",
+        templates={
+            "template1": TemplateConfig(
+                template_name="template1",
+                template_path="https://example.com/xyz/abcd.git",
+            ),
+            "template2": TemplateConfig(
+                template_name="template2",
+                template_path="https://example.com/git/example.git",
+            ),
+        },
+    )
+    example_config_path = pathlib.Path(__file__).parent.joinpath("example_config.yml")
+
+    def test_read_config(self):
+        with patch(
+            "data_pipelines_cli.data_structures.CONFIGURATION_PATH",
+            self.example_config_path,
+        ):
+            self.assertEqual(self.example_config_dict, read_config_or_exit())
+
+    def test_list_templates(self):
+        runner = CliRunner()
+        with patch(
+            "data_pipelines_cli.data_structures.CONFIGURATION_PATH",
+            self.example_config_path,
+        ):
+            result = runner.invoke(list_templates_command)
+            self.assertEqual(0, result.exit_code)
+            self.assertEqual(
+                "AVAILABLE TEMPLATES:\n\n"
+                "template_name: template1\n"
+                "template_path: https://example.com/xyz/abcd.git\n\n"
+                "template_name: template2\n"
+                "template_path: https://example.com/git/example.git\n\n",
+                result.output,
+            )
diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py
deleted file mode 100644
index 201975f..0000000
--- a/tests/test_placeholder.py
+++ /dev/null
@@ -1,2 +0,0 @@
-def test_placeholder():
-    pass