Repository: meltano/edk (language: python)

Issue:
chore: switch to namespaced package
closes #1 


Discussion / hints:

Fix patch:
diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000..cf3fdb5
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,7 @@
+[flake8]
+ignore = W503, C901, ANN101
+max-line-length = 88
+exclude = files_airflow_ext,cookiecutter
+max-complexity = 10
+docstring-convention = google
+allow-star-arg-any = true
diff --git a/.github/semantic.yml b/.github/semantic.yml
new file mode 100644
index 0000000..df55d53
--- /dev/null
+++ b/.github/semantic.yml
@@ -0,0 +1,31 @@
+# Config ref: https://github.com/Ezard/semantic-prs
+
+# Validate the PR title, and ignore all commit messages
+titleOnly: true
+
+# Provides a custom URL for the "Details" link, which appears next to the success/failure message from the app:
+targetUrl: https://github.com/meltano/sdk/blob/main/CONTRIBUTING.md#semantic-pull-requests
+
+# The values allowed for the "type" part of the PR title/commit message.
+# e.g. for a PR title/commit message of "feat: add some stuff", the type would be "feat"
+types:
+  - ci
+  - chore
+  - build
+  - docs
+  - feat
+  - fix
+  - perf
+  - refactor
+  - revert
+  - style
+  - test
+
+# The values allowed for the "scope" part of the PR title/commit message.
+# e.g. for a PR title/commit message of "feat(awesome-feature): add some stuff",
+#      the scope would be "awesome-feature"
+scopes:
+  - edk # edk only
+  - templates  # cookiecutter templates only
+  - deps       # production dependencies
+  - deps-dev   # development depencencies (testing, linting, etc.)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..68bbb5c
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,49 @@
+ci:
+  autofix_prs: false
+  autoupdate_schedule: weekly
+  autoupdate_commit_msg: 'chore: pre-commit autoupdate'
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: check-json
+  - id: check-toml
+    exclude: |
+        (?x)^(
+            cookiecutter/.*/pyproject.toml
+        )$
+  - id: end-of-file-fixer
+    exclude: (cookiecutter/.*|docs/.*|samples/.*\.json)
+  - id: trailing-whitespace
+
+- repo: https://github.com/psf/black
+  rev: 22.6.0
+  hooks:
+  - id: black
+    exclude: |
+        (?x)^(
+            cookiecutter/.*
+        )$
+
+- repo: https://github.com/pycqa/isort
+  rev: 5.10.1
+  hooks:
+  - id: isort
+    exclude: (cookiecutter/.*)
+
+- repo: https://github.com/pycqa/flake8
+  rev: 5.0.4
+  hooks:
+  - id: flake8
+    additional_dependencies:
+    - darglint==1.8.1
+    - flake8-annotations==2.9.0
+    - flake8-docstrings==1.6.0
+    files: 'meltano/edk/.*'
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: v2.37.3
+  hooks:
+  - id: pyupgrade
+    args: [--py37-plus]
diff --git a/meltano/edk/__init__.py b/meltano/edk/__init__.py
index b794fd4..091d55e 100644
--- a/meltano/edk/__init__.py
+++ b/meltano/edk/__init__.py
@@ -1,1 +1,1 @@
-__version__ = '0.1.0'
+"""The meltano extension development kit."""
diff --git a/meltano/edk/extension.py b/meltano/edk/extension.py
new file mode 100644
index 0000000..a53b5b3
--- /dev/null
+++ b/meltano/edk/extension.py
@@ -0,0 +1,130 @@
+"""Meltano extension SDK base class and supporting methods."""
+from __future__ import annotations
+
+import sys
+from abc import ABCMeta, abstractmethod
+from enum import Enum
+from typing import Any
+
+import structlog
+import yaml
+from devtools.prettier import pformat
+from meltano.edk import models
+
+
+class DescribeFormat(str, Enum):
+    """The currently supported Describe output formats."""
+
+    text = "text"
+    json = "json"
+    yaml = "yaml"
+
+
+class ExtensionBase(metaclass=ABCMeta):
+    """Basic extension interface that must be implemented by all extensions."""
+
+    def pre_invoke(self) -> None:
+        """Called before the extension is invoked."""
+        pass
+
+    def initialize(self, force: bool = False) -> None:
+        """Initialize the extension.
+
+        This method is called on-demand by the user to initialize the extension.
+        Extensions are not required to implement this method, and may no-op.
+
+        Args:
+            force: If True, force initialization.
+        """
+        pass
+
+    @abstractmethod
+    def invoke(self, command_name: str | None, *command_args: Any) -> None:
+        """Invoke method.
+
+        This method is called when the extension is invoked.
+
+        Args:
+            command_name: The name of the command to invoke.
+            *command_args: The arguments to pass to the command.
+        """
+        pass
+
+    def post_invoke(self) -> None:
+        """Called after the extension is invoked."""
+        pass
+
+    @abstractmethod
+    def describe(self) -> models.Describe:
+        """Describe method.
+
+        This method should describe what commands & capabilities the extension provides.
+
+        Returns:
+            Description: A description of the extension.
+        """
+        pass
+
+    def describe_formatted(
+        self, output_format: DescribeFormat = DescribeFormat.text
+    ) -> str:
+        """Return a formatted description of the extensions commands and capabilities.
+
+        Args:
+            output_format: The output format to use.
+
+        Returns:
+            str: The formatted description.
+        """
+        if output_format == DescribeFormat.text:
+            return pformat(self.describe())
+        elif output_format == DescribeFormat.json:
+            return self.describe().json(indent=2)
+        elif output_format == DescribeFormat.yaml:
+            # just calling describe().dict() and dumping that to yaml yields a yaml that
+            # is subtly different to the json variant in that it you have an additional
+            # level of nesting.
+            return yaml.dump(
+                yaml.safe_load(self.describe().json()), sort_keys=False, indent=2
+            )
+
+    def pass_through_invoker(
+        self, logger: structlog.BoundLogger, *command_args: Any
+    ) -> None:
+        """Pass-through invoker.
+
+        This method is used to invoke the wrapped CLI with arbitrary arguments.
+        Note this method will hard exit the process if an unhandled exception is
+        encountered.
+
+        Args:
+            logger: The logger to use in the event an exception needs to be logged.
+            *command_args: The arguments to pass to the command.
+        """
+        logger.debug(
+            "pass through invoker called",
+            command_args=command_args,
+        )
+        try:
+            self.pre_invoke()
+        except Exception:
+            logger.exception(
+                "pre_invoke failed with uncaught exception, please report to maintainer"
+            )
+            sys.exit(1)
+
+        try:
+            self.invoke(None, command_args)
+        except Exception:
+            logger.exception(
+                "invoke failed with uncaught exception, please report to maintainer"
+            )
+            sys.exit(1)
+
+        try:
+            self.post_invoke()
+        except Exception:
+            logger.exception(
+                "post_invoke failed with uncaught exception, please report to maintainer"  # noqa: E501
+            )
+            sys.exit(1)
diff --git a/meltano/edk/logging.py b/meltano/edk/logging.py
new file mode 100644
index 0000000..68d0d36
--- /dev/null
+++ b/meltano/edk/logging.py
@@ -0,0 +1,112 @@
+"""Recommended logging configuration for extensions."""
+from __future__ import annotations
+
+import logging
+import os
+import sys
+
+import structlog
+
+LEVELS = {  # noqa: WPS407
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+    "critical": logging.CRITICAL,
+}
+
+DEFAULT_LEVEL = "info"
+
+
+def parse_log_level(log_level: dict[str, int]) -> int:
+    """Parse a level descriptor into an logging level.
+
+    Args:
+        log_level: level descriptor.
+
+    Returns:
+        int: actual logging level.
+    """
+    return LEVELS.get(log_level.lower(), LEVELS[DEFAULT_LEVEL])
+
+
+def default_logging_config(
+    level: int = logging.INFO,
+    timestamps: bool = False,
+    levels: bool = False,
+    json_format: bool = False,
+) -> None:
+    """default/demo structlog configuration.
+
+    Args:
+        level: logging level.
+        timestamps: include timestamps in the log.
+        levels: include levels in the log.
+        json_format: if True, use JSON format, otherwise use human-readable format.
+    """
+    processors = []
+    if timestamps:
+        processors.append(structlog.processors.TimeStamper(fmt="iso"))
+    if levels:
+        processors.append(structlog.processors.add_log_level)
+
+    processors.extend(
+        [
+            # If log level is too low, abort pipeline and throw away log entry.
+            structlog.stdlib.filter_by_level,
+            # If the "stack_info" key in the event dict is true, remove it and
+            # render the current stack trace in the "stack" key.
+            structlog.processors.StackInfoRenderer(),
+            # If the "exc_info" key in the event dict is either true or a
+            # sys.exc_info() tuple, remove "exc_info" and render the exception
+            # with traceback into the "exception" key.
+            structlog.processors.format_exc_info,
+            # If some value is in bytes, decode it to a unicode str.
+            structlog.processors.UnicodeDecoder(),
+            structlog.processors.ExceptionPrettyPrinter(),
+            # Render the final event dict as JSON.
+            structlog.processors.JSONRenderer()
+            if json_format
+            else structlog.dev.ConsoleRenderer(colors=False),
+        ]
+    )
+
+    structlog.configure(
+        processors=processors,
+        # `wrapper_class` is the bound logger that you get back from
+        # get_logger(). This one imitates the API of `logging.Logger`.
+        wrapper_class=structlog.stdlib.BoundLogger,
+        # `logger_factory` is used to create wrapped loggers that are used for
+        # OUTPUT. This one returns a `logging.Logger`. The final value (a JSON
+        # string) from the final processor (`JSONRenderer`) will be passed to
+        # the method of the same name as that you've called on the bound logger.
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        # Effectively freeze configuration after creating the first bound
+        # logger.
+        cache_logger_on_first_use=True,
+    )
+
+    logging.basicConfig(
+        format="%(message)s",
+        stream=sys.stderr,
+        level=level,
+    )
+
+
+def pass_through_logging_config() -> None:
+    """Pass-through logging configuration.
+
+    Setups a logging config using the LOG_LEVEL, LOG_TIMESTAMPS, LOG_LEVELS,
+    and MELTANO_LOG_JSON env vars.
+    """
+    log_level = os.environ.get("LOG_LEVEL", "INFO")
+    log_timestamps = os.environ.get("LOG_TIMESTAMPS", False)
+    log_levels = os.environ.get("LOG_LEVELS", False)
+    meltano_log_json = os.environ.get("MELTANO_LOG_JSON", False)
+
+    default_logging_config(
+        level=parse_log_level(log_level),
+        timestamps=log_timestamps,
+        levels=log_levels,
+        json_format=meltano_log_json,
+    )
diff --git a/meltano/edk/models.py b/meltano/edk/models.py
new file mode 100644
index 0000000..d751315
--- /dev/null
+++ b/meltano/edk/models.py
@@ -0,0 +1,41 @@
+"""Various models used to describe extensions."""
+from typing import List
+
+from pydantic import BaseModel
+
+
+class Command(BaseModel):
+    """Describe a generic runnable command."""
+
+    name: str
+    description: str
+    commands: List[str] = []
+    pass_through_cli: bool = False
+
+
+class ExtensionCommand(Command):
+    """Describes an extension command."""
+
+    description = "The extension cli"
+    pass_through_cli: bool = False
+    commands: List[str] = [
+        "describe",
+        "invoke",
+        "pre_invoke",
+        "post_invoke",
+        "initialize",
+    ]
+
+
+class InvokerCommand(Command):
+    """Describes an invoker style command."""
+
+    description = "The pass through invoker cli"
+    pass_through_cli: bool = True
+    commands: List[str] = [":splat"]
+
+
+class Describe(BaseModel):
+    """Describes what commands and capabilities the extension provides."""
+
+    commands: List[Command]
diff --git a/meltano/edk/process.py b/meltano/edk/process.py
new file mode 100644
index 0000000..1c84c1e
--- /dev/null
+++ b/meltano/edk/process.py
@@ -0,0 +1,152 @@
+"""Utilities for working with subprocesses."""
+from __future__ import annotations
+
+import asyncio
+import os
+import subprocess
+from typing import IO, Any
+
+import structlog
+
+log = structlog.get_logger()
+
+
+def log_subprocess_error(
+    cmd: str, err: subprocess.CalledProcessError, error_message: str
+) -> None:
+    """Log a subprocess error, replaying stderr to the logger if it's available.
+
+    Args:
+        cmd: the command that was run.
+        err: the error that was raised.
+        error_message: the error message to log.
+    """
+    if err.stderr:
+        for line in err.stderr.split("\n"):
+            log.warning(line, cmd=cmd, stdio_stream="stderr")
+    log.error(
+        f"error invoking {cmd}",
+        returncode=err.returncode,
+        error_message=error_message,
+    )
+
+
+class Invoker:
+    """Invoker utility class for invoking subprocesses."""
+
+    def __init__(
+        self,
+        bin: str,
+        cwd: str = None,
+        env: dict[str, any] | None = None,
+    ) -> None:
+        """Minimal invoker for running subprocesses.
+
+        Args:
+            bin: The path/name of the binary to run.
+            cwd: The working directory to run from.
+            env: Env to use when calling Popen, defaults to current os.environ if None.
+        """
+        self.bin = bin
+        self.cwd = cwd
+        self.popen_env = env if env else os.environ.copy()
+
+    def run(
+        self,
+        *args: str | bytes | os.PathLike[str] | os.PathLike[bytes],
+        stdout: None | int | IO = subprocess.PIPE,
+        stderr: None | int | IO = subprocess.PIPE,
+        text: bool = True,
+        **kwargs: Any,
+    ) -> subprocess.CompletedProcess:
+        """Run a subprocess. Simple wrapper around subprocess.run.
+
+        Note that output from stdout and stderr is NOT logged automatically. Especially
+        useful when you want to run a command, but don't care about its output and only
+        care about its return code.
+
+        stdout and stderr by default are set up to use subprocess.PIPE. If you do not
+        want to capture io from the subprocess use subprocess.DEVNULL to discard it.
+
+        Args:
+            *args: The arguments to pass to the subprocess.
+            stdout: The stdout stream to use.
+            stderr: The stderr stream to use.
+            text: If true, decode stdin, stdout and stderr using the system default.
+            **kwargs: Additional keyword arguments to pass to subprocess.run.
+
+        May indirectly raise CalledProcessError if the underlying subprocess failed.
+
+        Returns:
+            The completed process.
+        """
+        return subprocess.run(
+            [self.bin, *args],
+            cwd=self.cwd,
+            stdout=stdout,
+            stderr=stderr,
+            check=True,
+            text=text,
+            **kwargs,
+        )
+
+    @staticmethod
+    async def _log_stdio(reader: asyncio.streams.StreamReader) -> None:
+        """Log the output of a stream.
+
+        Args:
+            reader: The stream reader to read from.
+        """
+        while True:
+            if reader.at_eof():
+                break
+            data = await reader.readline()
+            log.info(data.decode("utf-8").rstrip())
+            await asyncio.sleep(0)
+
+    async def _exec(
+        self,
+        sub_command: str | None = None,
+        *args: str | bytes | os.PathLike[str] | os.PathLike[bytes],
+    ) -> asyncio.subprocess.Process:
+        popen_args = []
+        if sub_command:
+            popen_args.append(sub_command)
+        if args:
+            popen_args.extend(*args)
+
+        p = await asyncio.create_subprocess_exec(
+            self.bin,
+            *popen_args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=self.popen_env,
+        )
+        asyncio.create_task(self._log_stdio(p.stderr))
+        asyncio.create_task(self._log_stdio(p.stdout))
+
+        await p.wait()
+        return p
+
+    def run_and_log(
+        self,
+        sub_command: str | None = None,
+        *args: str | bytes | os.PathLike[str] | os.PathLike[bytes],
+    ) -> None:
+        """Run a subprocess and stream the output to the logger.
+
+        Note that output from stdout and stderr IS logged. Best used when you want
+        to run a command and stream the output to a user.
+
+        Args:
+            sub_command: The subcommand to run.
+            *args: The arguments to pass to the subprocess.
+
+        Raises:
+            CalledProcessError: If the subprocess failed.
+        """
+        result = asyncio.run(self._exec(sub_command, *args))
+        if result.returncode:
+            raise subprocess.CalledProcessError(
+                result.returncode, cmd=self.bin, stderr=None
+            )
diff --git a/pyproject.toml b/pyproject.toml
index 8721fb3..85a6a45 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,10 +42,24 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "<3.11,>=3.7"
+structlog = "^21.2.0"
+PyYAML = "^6.0.0"
+pydantic = "^1.9.0"
+devtools = "^0.9.0"
 
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"
+black = "^22.3.0"
+isort = "^5.10.1"
+flake8 = "^3.9.0"
+
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3 # Vertical Hanging Indent
+src_paths = "ext_airflow"
+
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]


Test patch:
diff --git a/tests/test_meltano_edk.py b/tests/test_meltano_edk.py
index 08a128e..f19b44a 100644
--- a/tests/test_meltano_edk.py
+++ b/tests/test_meltano_edk.py
@@ -1,5 +1,21 @@
-from meltano_edk import __version__
+from __future__ import annotations
 
+from meltano.edk import models
+from meltano.edk.extension import ExtensionBase
 
-def test_version():
-    assert __version__ == '0.1.0'
+
+class TestExtension(ExtensionBase):
+    def invoke(self, command_name: str | None, *command_args) -> None:
+        pass
+
+    def describe(self) -> models.Describe:
+        return models.Describe(
+            commands=[models.ExtensionCommand(name="test", description="test")]
+        )
+
+
+def test_canary():
+    test = TestExtension()
+    assert test.describe() == models.Describe(
+        commands=[models.ExtensionCommand(name="test", description="test")]
+    )