Repository: JonathonReinhart/scuba
Pull request #192 (2022-01-10 00:31:07)

Expand env variables in volume paths in configuration file
For more context on this change, see #191.

github-actions[bot]: ## Unit Test Results
    5 files      5 suites   1m 51s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests")
153 tests 150 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests")   2 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") 1 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests")
765 runs  750 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") 10 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") 5 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests")

For more details on these failures, see [this check](https://github.com/JonathonReinhart/scuba/runs/4755895476).

Results for commit 3a85fce1.


Patch:
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 7b522fd..5f35c0c 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -166,6 +166,19 @@ of volume options:
         hostpath: /host/foo
         options: ro,cached
 
+The keys used in volume mappings can contain environment variables **that are
+expanded in the host environment**. For example, this configuration would map
+the user's ``/home/username/.config/application1`` directory into the container
+at the same path.
+
+.. code-block:: yaml
+
+    volumes:
+      $TEST_HOME/.config/application1: $TEST_HOME/.config/application1
+
+Note that because variable expansion is now applied to all volume keys, if one
+desires to have a key with an explicit ``$`` character, it must be written as
+``$$``.
 
 
 .. _conf_aliases:
diff --git a/scuba/config.py b/scuba/config.py
index 45039d6..603a261 100644
--- a/scuba/config.py
+++ b/scuba/config.py
@@ -261,9 +261,22 @@ def _get_volumes(data):
 
     vols = {}
     for cpath, v in voldata.items():
+        cpath = _expand_path(cpath)
         vols[cpath] = ScubaVolume.from_dict(cpath, v)
     return vols
 
+def _expand_path(in_str):
+    try:
+        output = expand_env_vars(in_str)
+    except KeyError as ke:
+        # pylint: disable=raise-missing-from
+        raise ConfigError("Unset environment variable '{}' used in '{}'".format(ke.args[0], in_str))
+    except ValueError as ve:
+        raise ConfigError("Unable to expand string '{}' due to parsing "
+                "errors".format(in_str)) from ve
+
+    return output
+
 class ScubaVolume:
     def __init__(self, container_path, host_path=None, options=None):
         self.container_path = container_path
@@ -282,7 +295,7 @@ class ScubaVolume:
         if isinstance(node, str):
             return cls(
                 container_path = cpath,
-                host_path = node,
+                host_path = _expand_path(node),
                 )
 
         # Complex form
@@ -296,7 +309,7 @@ class ScubaVolume:
                 raise ConfigError("Volume {} must have a 'hostpath' subkey".format(cpath))
             return cls(
                 container_path = cpath,
-                host_path = hpath,
+                host_path = _expand_path(hpath),
                 options = _get_delimited_str_list(node, 'options', ','),
                 )
 
diff --git a/scuba/utils.py b/scuba/utils.py
index 8a62978..8159aed 100644
--- a/scuba/utils.py
+++ b/scuba/utils.py
@@ -1,6 +1,7 @@
 import errno
 import os
 from shlex import quote as shell_quote
+import string
 
 
 def shell_quote_cmd(cmdlist):
@@ -79,3 +80,10 @@ def get_umask():
 
 def writeln(f, line):
     f.write(line + '\n')
+
+def expand_env_vars(in_str):
+    """Expand environment variables in a string
+
+    Can raise `KeyError` if a variable is referenced but not defined, similar to
+    bash's nounset (set -u) option"""
+    return string.Template(in_str).substitute(os.environ)


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
index 568f041..c711f30 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -883,3 +883,76 @@ class TestConfig:
         assert v.container_path == '/bar'
         assert v.host_path == '/host/bar'
         assert v.options == ['z', 'ro']
+
+    def test_volumes_with_env_vars_simple(self, monkeypatch):
+        '''volume definitions can contain environment variables'''
+        monkeypatch.setenv("TEST_VOL_PATH", "/bar/baz")
+        monkeypatch.setenv("TEST_VOL_PATH2", "/moo/doo")
+        with open('.scuba.yml', 'w') as f:
+            f.write(r'''
+                image: na
+                volumes:
+                  $TEST_VOL_PATH/foo: ${TEST_VOL_PATH2}/foo
+                ''')
+
+        config = scuba.config.load_config('.scuba.yml')
+        vols = config.volumes
+        assert len(vols) == 1
+
+        v = list(vols.values())[0]
+        assert isinstance(v, scuba.config.ScubaVolume)
+        assert v.container_path == '/bar/baz/foo'
+        assert v.host_path == '/moo/doo/foo'
+        assert v.options == []
+
+    def test_volumes_with_env_vars_complex(self, monkeypatch):
+        '''complex volume definitions can contain environment variables'''
+        monkeypatch.setenv("TEST_HOME", "/home/testuser")
+        monkeypatch.setenv("TEST_TMP", "/tmp")
+        monkeypatch.setenv("TEST_MAIL", "/var/spool/mail/testuser")
+
+        with open('.scuba.yml', 'w') as f:
+            f.write(r'''
+                image: na
+                volumes:
+                  $TEST_HOME/.config: ${TEST_HOME}/.config
+                  $TEST_TMP/:
+                    hostpath: $TEST_HOME/scuba/myproject/tmp
+                  /var/spool/mail/container:
+                    hostpath: $TEST_MAIL
+                    options: z,ro
+                ''')
+
+        config = scuba.config.load_config('.scuba.yml')
+        vols = config.volumes
+        assert len(vols) == 3
+
+        v = vols['/home/testuser/.config']
+        assert isinstance(v, scuba.config.ScubaVolume)
+        assert v.container_path == '/home/testuser/.config'
+        assert v.host_path == '/home/testuser/.config'
+        assert v.options == []
+
+        v = vols['/tmp/']
+        assert isinstance(v, scuba.config.ScubaVolume)
+        assert v.container_path == '/tmp/'
+        assert v.host_path == '/home/testuser/scuba/myproject/tmp'
+        assert v.options == []
+
+        v = vols['/var/spool/mail/container']
+        assert isinstance(v, scuba.config.ScubaVolume)
+        assert v.container_path == '/var/spool/mail/container'
+        assert v.host_path == "/var/spool/mail/testuser"
+        assert v.options == ['z', 'ro']
+
+    def test_volumes_with_invalid_env_vars(self, monkeypatch):
+        '''Volume definitions cannot include unset env vars'''
+        # Ensure that the entry does not exist in the environment
+        monkeypatch.delenv("TEST_VAR1", raising=False)
+        with open('.scuba.yml', 'w') as f:
+            f.write(r'''
+                image: na
+                volumes:
+                  $TEST_VAR1/foo: /host/foo
+                ''')
+        self._invalid_config('TEST_VAR1')
diff --git a/tests/test_main.py b/tests/test_main.py
index 78bb848..3d9865e 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -1,6 +1,7 @@
 from .utils import *
 from unittest import mock
 import pytest
+import warnings
 
 import logging
 import os
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 243f3a5..d8075e5 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -128,3 +128,26 @@ def test_writeln():
         scuba.utils.writeln(s, 'hello')
         scuba.utils.writeln(s, 'goodbye')
         assert s.getvalue() == 'hello\ngoodbye\n'
+
+def test_expand_env_vars(monkeypatch):
+    monkeypatch.setenv("MY_VAR", "my favorite variable")
+    assert scuba.utils.expand_env_vars("This is $MY_VAR") == \
+            "This is my favorite variable"
+    assert scuba.utils.expand_env_vars("What is ${MY_VAR}?") == \
+            "What is my favorite variable?"
+
+def test_expand_missing_env_vars(monkeypatch):
+    monkeypatch.delenv("MY_VAR", raising=False)
+    # Verify that a KeyError is raised for unset env variables
+    with pytest.raises(KeyError) as kerr:
+        scuba.utils.expand_env_vars("Where is ${MY_VAR}?")
+    assert kerr.value.args[0] == "MY_VAR"
+
+
+def test_expand_env_vars_dollars():
+    # Verify that a ValueError is raised for bare, unescaped '$' characters
+    with pytest.raises(ValueError):
+        scuba.utils.expand_env_vars("Just a lonely $")
+
+    # Verify that it is possible to get '$' characters in an expanded string
+    assert scuba.utils.expand_env_vars(r"Just a lonely $$") == "Just a lonely $"