Repository: metricq/metricq-tools (language: python)

Issue:
Allow parsing arguments from environment variables and .env files
click & dotenv ftw


Discussion / hints:

Fix patch:
diff --git a/metricq_tools/__init__.py b/metricq_tools/__init__.py
index 432a187..9c9c5af 100644
--- a/metricq_tools/__init__.py
+++ b/metricq_tools/__init__.py
@@ -1,25 +1,3 @@
-# metricq-tools
-# Copyright (C) 2021 ZIH, Technische Universitaet Dresden, Federal Republic of Germany
-#
-# All rights reserved.
-#
-# This file is part of metricq.
-#
-# metricq is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# metricq is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with metricq.  If not, see <http://www.gnu.org/licenses/>.
-
-# pylint: disable=logging-fstring-interpolation,logging-format-interpolation
-
 from . import discover, inspect, send, spy
 from .version import version as __version__
 
diff --git a/metricq_tools/check.py b/metricq_tools/check.py
index 434fa3f..89982ac 100644
--- a/metricq_tools/check.py
+++ b/metricq_tools/check.py
@@ -46,7 +46,7 @@ async def check_metric(metric: str) -> None:
         print(metric, aggregate)
 
 
-@metricq_command(default_token="history-tool-check")
+@metricq_command(default_token="history-$USER-tool-check")
 def main(server: str, token: str) -> None:
     """Check all historic metrics for non-finite values."""
     client = metricq.HistoryClient(
diff --git a/metricq_tools/discover.py b/metricq_tools/discover.py
index 5de20a5..fe5b8c4 100755
--- a/metricq_tools/discover.py
+++ b/metricq_tools/discover.py
@@ -1,32 +1,3 @@
-#!/usr/bin/env python3
-# Copyright (c) 2020, ZIH, Technische Universitaet Dresden, Federal Republic of Germany
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#     * Redistributions of source code must retain the above copyright notice,
-#       this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright notice,
-#       this list of conditions and the following disclaimer in the documentation
-#       and/or other materials provided with the distribution.
-#     * Neither the name of metricq nor the names of its contributors
-#       may be used to endorse or promote products derived from this software
-#       without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 import asyncio
 import datetime
 import json
@@ -343,7 +314,7 @@ async def discover(
                     echo_status(Status.Error, from_token, str(response))
 
 
-@metricq_command(default_token="agent-tool-discover")
+@metricq_command(default_token="agent-$USER-tool-discover")
 @click.option(
     "-d",
     "--diff",
diff --git a/metricq_tools/inspect.py b/metricq_tools/inspect.py
index 130151d..decd431 100755
--- a/metricq_tools/inspect.py
+++ b/metricq_tools/inspect.py
@@ -1,32 +1,3 @@
-#!/usr/bin/env python3
-# Copyright (c) 2019, ZIH, Technische Universitaet Dresden, Federal Republic of Germany
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#     * Redistributions of source code must retain the above copyright notice,
-#       this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright notice,
-#       this list of conditions and the following disclaimer in the documentation
-#       and/or other materials provided with the distribution.
-#     * Neither the name of metricq nor the names of its contributors
-#       may be used to endorse or promote products derived from this software
-#       without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 from contextlib import suppress
 from typing import Any, Optional
 
@@ -37,9 +8,8 @@
 import termplotlib as tpl  # type: ignore
 from metricq.datachunk_pb2 import DataChunk
 
-from metricq_tools.utils import metricq_command
-
 from .logging import logger
+from .utils import metricq_command
 from .version import version as client_version
 
 
@@ -214,7 +184,7 @@ def print_histograms(self) -> None:
             self.print_values_histogram()
 
 
-@metricq_command(default_token="agent-tool-inspect")
+@metricq_command(default_token="agent-$USER-tool-inspect")
 @click.option(
     "--intervals-histogram/--no-intervals-histogram",
     "-i/-I",
diff --git a/metricq_tools/logging.py b/metricq_tools/logging.py
index d3926fd..e55199e 100644
--- a/metricq_tools/logging.py
+++ b/metricq_tools/logging.py
@@ -1,32 +1,3 @@
-#!/usr/bin/env python3
-# Copyright (c) 2021, ZIH, Technische Universitaet Dresden, Federal Republic of Germany
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#     * Redistributions of source code must retain the above copyright notice,
-#       this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright notice,
-#       this list of conditions and the following disclaimer in the documentation
-#       and/or other materials provided with the distribution.
-#     * Neither the name of metricq nor the names of its contributors
-#       may be used to endorse or promote products derived from this software
-#       without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 import logging
 
 import click_log  # type: ignore
diff --git a/metricq_tools/send.py b/metricq_tools/send.py
index 6201dec..0335d2a 100755
--- a/metricq_tools/send.py
+++ b/metricq_tools/send.py
@@ -1,38 +1,9 @@
-#!/usr/bin/env python3
-# Copyright (c) 2020, ZIH, Technische Universitaet Dresden, Federal Republic of Germany
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#     * Redistributions of source code must retain the above copyright notice,
-#       this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright notice,
-#       this list of conditions and the following disclaimer in the documentation
-#       and/or other materials provided with the distribution.
-#     * Neither the name of metricq nor the names of its contributors
-#       may be used to endorse or promote products derived from this software
-#       without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 from typing import Any
 
 import click
 from metricq import Metric, Source, Timestamp
 
-from metricq_tools.utils import TIMESTAMP, metricq_command
-
+from .utils import TIMESTAMP, metricq_command
 from .version import version as client_version
 
 
@@ -50,7 +21,7 @@ async def task(self) -> None:
         await self.stop()
 
 
-@metricq_command(default_token="source-tool-send")
+@metricq_command(default_token="source-$USER-tool-send")
 @click.option(
     "--timestamp",
     type=TIMESTAMP,
diff --git a/metricq_tools/spy.py b/metricq_tools/spy.py
index ab0555f..9b89d86 100755
--- a/metricq_tools/spy.py
+++ b/metricq_tools/spy.py
@@ -1,32 +1,3 @@
-#!/usr/bin/env python3
-# Copyright (c) 2020, ZIH, Technische Universitaet Dresden, Federal Republic of Germany
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#     * Redistributions of source code must retain the above copyright notice,
-#       this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright notice,
-#       this list of conditions and the following disclaimer in the documentation
-#       and/or other materials provided with the distribution.
-#     * Neither the name of metricq nor the names of its contributors
-#       may be used to endorse or promote products derived from this software
-#       without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 import asyncio
 from contextlib import suppress
 from typing import Any, Dict, Optional, TypedDict
@@ -126,7 +97,7 @@ async def _on_history_response(
         await super()._on_history_response(message)
 
 
-@metricq_command(default_token="agent-tool-spy")
+@metricq_command(default_token="agent-$USER-tool-spy")
 @output_format_option()
 @click.argument("metrics", required=True, nargs=-1)
 def main(server: str, token: str, format: OutputFormat, metrics: list[str]) -> None:
diff --git a/metricq_tools/summary.py b/metricq_tools/summary.py
index e1313ec..ca487a7 100755
--- a/metricq_tools/summary.py
+++ b/metricq_tools/summary.py
@@ -1,33 +1,3 @@
-#!/usr/bin/env python3
-# Copyright (c) 2021, ZIH, Technische Universitaet Dresden, Federal Republic of Germany
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#     * Redistributions of source code must retain the above copyright notice,
-#       this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright notice,
-#       this list of conditions and the following disclaimer in the documentation
-#       and/or other materials provided with the distribution.
-#     * Neither the name of metricq nor the names of its contributors
-#       may be used to endorse or promote products derived from this software
-#       without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 import asyncio
 from sys import exit
 from typing import Any, Optional
@@ -39,9 +9,8 @@
 from metricq import Subscriber
 from tabulate import tabulate
 
-from metricq_tools.utils import metricq_command
-
 from .logging import logger
+from .utils import TemplateStringParam, metricq_command
 from .version import version as client_version
 
 
@@ -260,7 +229,7 @@ async def async_main(
         return returncode
 
 
-@metricq_command(default_token="sink-tool-summary")
+@metricq_command(default_token="sink-$USER-tool-summary")
 @click.option(
     "--intervals-histogram/--no-intervals-histogram",
     "-i/-I",
@@ -275,7 +244,9 @@ async def async_main(
 )
 @click.option("--print-data-points/--no-print-data-points", "-d/-D", default=False)
 @click.option("--print-statistics/--no-print-statistics", "-s/-S", default=True)
-@click.option("-m", "--metric", required=True, multiple=True)
+@click.option(
+    "-m", "--metric", type=TemplateStringParam(), required=True, multiple=True
+)
 @click.argument("command", required=True, nargs=-1)
 def main(
     server: str,
diff --git a/metricq_tools/utils.py b/metricq_tools/utils.py
index d5668c8..3e9431e 100644
--- a/metricq_tools/utils.py
+++ b/metricq_tools/utils.py
@@ -1,10 +1,15 @@
 import re
+from contextlib import suppress
 from enum import Enum, auto
+from getpass import getuser
+from socket import gethostname
+from string import Template
 from typing import Any, Callable, Generic, List, Optional, Type, TypeVar, Union, cast
 
 import click
 import click_log  # type: ignore
 from click import Context, Parameter, ParamType, option
+from dotenv import find_dotenv, load_dotenv
 from metricq import Timedelta, Timestamp
 
 from .logging import logger
@@ -12,6 +17,15 @@
 
 _C = TypeVar("_C", covariant=True)
 
+# We do not interpolate (i.e. replace ${VAR} with corresponding environment variables).
+# That is because we want to be able to interpolate ourselves for metrics and tokens
+# using the same syntax. If it was only ${USER} for the token, we could use the
+# override functionality, but most unfortunately there is no standard environment
+# variable for the hostname. Even $HOST on zsh is not actually part of the environment.
+# ``override=false`` just means that environment variables have priority over the
+# env files.
+load_dotenv(dotenv_path=find_dotenv(".metricq"), interpolate=False, override=False)
+
 
 def camelcase_to_kebabcase(camelcase: str) -> str:
     # Match empty string preceeding uppercase character, but not at the start
@@ -151,9 +165,29 @@ def convert(
 TIMESTAMP = TimestampParam()
 
 
+class TemplateStringParam(ParamType):
+    name = "text"
+    mapping: dict[str, str]
+
+    def __init__(self):
+        self.mapping = {}
+        with suppress(Exception):
+            self.mapping["USER"] = getuser()
+        with suppress(Exception):
+            self.mapping["HOST"] = gethostname()
+
+    def convert(
+        self, value: Any, param: Optional[Parameter], ctx: Optional[Context]
+    ) -> str:
+        if not isinstance(value, str):
+            raise TypeError("expected a string type for TemplateStringParam")
+        return Template(value).safe_substitute(self.mapping)
+
+
 def metricq_server_option() -> Callable[[FC], FC]:
     return option(
         "--server",
+        type=TemplateStringParam(),
         metavar="URL",
         default="amqp://localhost/",
         show_default=True,
@@ -164,6 +198,7 @@ def metricq_server_option() -> Callable[[FC], FC]:
 def metricq_token_option(default: str) -> Callable[[FC], FC]:
     return option(
         "--token",
+        type=TemplateStringParam(),
         metavar="CLIENT_TOKEN",
         default=default,
         show_default=True,
@@ -175,12 +210,27 @@ def metricq_command(default_token: str) -> Callable[[FC], click.Command]:
     log_decorator = cast(
         Callable[[FC], FC], click_log.simple_verbosity_option(logger, default="warning")
     )
+    context_settings = {"auto_envvar_prefix": "METRICQ"}
+    epilog = (
+        "All options can be passed as environment variables prefixed with 'METRICQ_'."
+        "I.e., 'METRICQ_SERVER=amqps://...'.\n"
+        "\n"
+        "You can also create a '.metricq' file in the current or home directory that "
+        "contains environment variable settings in the same format.\n"
+        "\n"
+        "Some options, including server and token, can contain placeholders for $USER "
+        "and $HOST."
+    )
 
     def decorator(func: FC) -> click.Command:
         return click.version_option(version=client_version)(
             log_decorator(
                 metricq_token_option(default_token)(
-                    metricq_server_option()(click.command()(func))
+                    metricq_server_option()(
+                        click.command(context_settings=context_settings, epilog=epilog)(
+                            func
+                        )
+                    )
                 )
             )
         )
diff --git a/setup.cfg b/setup.cfg
index 6d476df..bc081d8 100755
--- a/setup.cfg
+++ b/setup.cfg
@@ -23,6 +23,7 @@ install_requires =
     async-timeout~=3.0
     humanize~=2.5
     python-dateutil~=2.8
+    python-dotenv~=1.0.0
     numpy
     termplotlib
     tabulate


Test patch:
diff --git a/tests/test_params.py b/tests/test_params.py
index cabbb27..e534d92 100644
--- a/tests/test_params.py
+++ b/tests/test_params.py
@@ -1,3 +1,5 @@
+import getpass
+import socket
 from enum import Enum, auto
 from typing import Union
 
@@ -10,6 +12,7 @@
     ChoiceParam,
     CommandLineChoice,
     DurationParam,
+    TemplateStringParam,
     TimestampParam,
 )
 
@@ -70,3 +73,20 @@ def test_duration_param() -> None:
     value = "30s"
     DURATION = DurationParam(default=None)
     assert DURATION.convert(value, param=None, ctx=None) == Timedelta.from_string(value)
+
+
+def test_template_string_param_user() -> None:
+    value = "foo-$USER-bar"
+    ref = f"foo-{getpass.getuser()}-bar"
+    assert TemplateStringParam()(value, param=None, ctx=None) == ref
+
+
+def test_template_string_param_host() -> None:
+    value = "foo-$HOST-bar"
+    ref = f"foo-{socket.gethostname()}-bar"
+    assert TemplateStringParam()(value, param=None, ctx=None) == ref
+
+
+def test_template_string_param_unknown() -> None:
+    value = "foo-$NOONEWILLEVERSETTHISVARIABLE"
+    assert TemplateStringParam()(value, param=None, ctx=None) == value