Repository: peopledoc/vault-cli

Issue:
vault-cli set-all
Load multiple keys at once from a format compatible with (flat) get-all.

This can be useful to move secrets between two vault clusters:
```console
$ vault --config-file=a.yml get-all | vault --config-file=b.yml set-all
```

I'm guessing there would be a single optional positional argument for the file which defaults to `-`, interpreted as `stdin`.
Yaml content should be checked for format: it should be a valid yaml, containing an object with string keys and each value is itself an object.
We should then defer individual writes to `set_value` and provide the same set of flags to handle safe/unsafe overwriting.

Discussion:

Patch:
diff --git a/docs/howto/import_export.rst b/docs/howto/import_export.rst
new file mode 100644
index 0000000..f5795d4
--- /dev/null
+++ b/docs/howto/import_export.rst
@@ -0,0 +1,14 @@
+Move secrets from a Vault cluster to a different Vault cluster
+==============================================================
+
+The combination of ``vault-cli get-all`` and ``vault-cli set-all`` gives you a
+handy way to move data between two vaults. Let's consider you have prepared
+two configuration files ``source.yml`` and ``dest.yml`` containing connection
+parameters (see `configure`).
+
+.. code:: console
+
+    $ vault-cli --config-file=source.yml get-all \
+        | vault-cli --config-file=dest.yml set-all
+
+That's it.
diff --git a/docs/howto/write.rst b/docs/howto/write.rst
index b32b705..df82ef4 100644
--- a/docs/howto/write.rst
+++ b/docs/howto/write.rst
@@ -96,6 +96,33 @@ history nor on your screen.
    Please enter a value for key `mykey` of `mypath`:
    Done
 
+
+Write multiple secrets at once
+------------------------------
+
+Using ``vault-cli set-all`` will let you load multiple secrets from a ``yaml`` file
+(or ``yaml``-encoded ``stdin`` by default). The expected format is a mapping
+of secret paths and secret value objects.
+
+.. code:: console
+
+   $ vault-cli set-all
+   ---
+   a/b:
+      c: d
+   e:
+      f: g
+      h: i
+   <hit ctrl+d to end stdin>
+   Done
+   $ vault-cli get-all
+   ---
+   a/b:
+      c: d
+   e:
+      f: g
+      h: i
+
 Caveats
 -------
 
diff --git a/docs/howto_index.rst b/docs/howto_index.rst
index a016cc3..7a1f24c 100644
--- a/docs/howto_index.rst
+++ b/docs/howto_index.rst
@@ -20,6 +20,7 @@ How-to...
     howto/permissions
     howto/safe_write
     howto/secret_with_dash
+    howto/import_export
     howto/library
     howto/test
     howto/troubleshooting
diff --git a/setup.cfg b/setup.cfg
index 37b15e0..ab61322 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -76,7 +76,7 @@ testpaths =
     tests/integration
 junit_family=xunit2
 
-[mypy-backports.functools_lru_cache.*,setuptools.*,urllib3.*,hvac.*,pytest.*]
+[mypy-backports.functools_lru_cache.*,setuptools.*,pytest.*]
 ignore_missing_imports = True
 
 [flake8]
diff --git a/vault_cli/cli.py b/vault_cli/cli.py
index ef0c45d..0ddcba8 100644
--- a/vault_cli/cli.py
+++ b/vault_cli/cli.py
@@ -378,9 +378,60 @@ def set_(
     try:
         client_obj.set_secret(path=path, value=json_value, force=force, update=update)
     except exceptions.VaultOverwriteSecretError as exc:
-        raise click.ClickException(
-            f"Secret already exists at {exc.path}. Use -f to force overwriting."
-        )
+        raise click.ClickException(f"{exc}\nUse -f to force overwriting")
+    except exceptions.VaultMixSecretAndFolder as exc:
+        raise click.ClickException(str(exc))
+    click.echo("Done")
+
+
+@cli.command("set-all")
+@click.pass_obj
+@click.option(
+    "--update/--clear",
+    default=True,
+    help="Update the current kv mapping or replace the its content",
+)
+@click.option(
+    "--force/--no-force",
+    "-f",
+    is_flag=True,
+    default=None,
+    help="In case the path already holds a secret, allow overwriting it "
+    "(this is necessary only if --safe-write is set).",
+)
+@click.argument(
+    "yaml_file",
+    default="-",
+    type=click.File(),
+)
+@handle_errors()
+def set_all(
+    client_obj: client.VaultClientBase,
+    update: bool,
+    force: Optional[bool],
+    yaml_file: TextIO,
+):
+    """
+    Set multiple secrets at once from a yaml mapping.
+    \b
+    Mapping expected format:
+    path/to/secret/a:
+        somekey: mysecret
+        otherkey: othersecret
+    path/to/secret/a:
+        yetanotherkey: [the, secret, is, a, list]
+    """
+    secrets = yaml.safe_load(yaml_file)
+    error = "Mapping expected format is a mapping of paths to secret objects"
+    if not isinstance(secrets, dict) or not all(
+        isinstance(v, dict) for v in secrets.values()
+    ):
+        raise click.ClickException(error)
+
+    try:
+        client_obj.set_secrets(secrets=secrets, force=force, update=update)
+    except exceptions.VaultOverwriteSecretError as exc:
+        raise click.ClickException(f"{exc}\nUse -f to force overwriting")
     except exceptions.VaultMixSecretAndFolder as exc:
         raise click.ClickException(str(exc))
     click.echo("Done")
@@ -544,9 +595,7 @@ def mv(
         ):
             click.echo(f"Move '{old_path}' to '{new_path}'")
     except exceptions.VaultOverwriteSecretError as exc:
-        raise click.ClickException(
-            f"Secret already exists at {exc.path}. Use -f to force overwriting."
-        )
+        raise click.ClickException(f"{exc}\nUse -f to force overwriting")
     except exceptions.VaultMixSecretAndFolder as exc:
         raise click.ClickException(str(exc))
 
@@ -576,9 +625,7 @@ def cp(
         ):
             click.echo(f"Copy '{old_path}' to '{new_path}'")
     except exceptions.VaultOverwriteSecretError as exc:
-        raise click.ClickException(
-            f"Secret already exists at {exc.path}. Use -f to force overwriting."
-        )
+        raise click.ClickException(f"{exc}\nUse -f to force overwriting")
     except exceptions.VaultMixSecretAndFolder as exc:
         raise click.ClickException(str(exc))
 
diff --git a/vault_cli/client.py b/vault_cli/client.py
index b0e39c3..17bd9c1 100644
--- a/vault_cli/client.py
+++ b/vault_cli/client.py
@@ -4,7 +4,7 @@ import logging
 import pathlib
 from typing import Dict, Iterable, List, Optional, Set, Tuple, Type, Union, cast
 
-import hvac
+import hvac  # type: ignore
 import jinja2
 import requests.packages.urllib3
 
@@ -639,15 +639,16 @@ class VaultClientBase:
             # if we update the mapping, we only have to check the updated keys
             if not update and not force and existing_value != value:
                 raise exceptions.VaultOverwriteSecretError(path=path)
-            if (
-                update
-                and not force
-                and any(
-                    existing_value[key] != value[key]
-                    for key in value.keys() & existing_value.keys()
-                )
-            ):
-                raise exceptions.VaultOverwriteSecretError(path=path)
+            if update and not force:
+                redefined = [
+                    key
+                    for key in (value.keys() & existing_value.keys())
+                    if existing_value[key] != value[key]
+                ]
+                if any(redefined):
+                    raise exceptions.VaultOverwriteSecretError(
+                        path=path, keys=redefined
+                    )
 
             if update:
                 # merge value with existing_value
@@ -683,6 +684,33 @@ class VaultClientBase:
 
         self._set_secret(path=self._build_full_path(path), secret=value)
 
+    def set_secrets(
+        self,
+        secrets: Dict[str, JSONDictRecursive],
+        force: Optional[bool] = None,
+        update: Optional[bool] = None,
+    ) -> None:
+        """
+        Sets the value of multiple secrets at once. See possible exceptions
+        in
+
+        Parameters
+        ----------
+        secrets :
+            A mapping of secret path -> value, corresponding to the output
+            of `VaultClientBase.get_secrets`
+        force : Optional[bool], optional
+            If safe_mode is True, whether to overwrite existing secrets
+        update: Optional[bool], optional
+            If true then merge the value with the existing one, else overwrite it
+
+        Raises
+        ------
+        see `VaultClientBase.set_secrets`
+        """
+        for path, value in secrets.items():
+            self.set_secret(path=path, value=value, force=force, update=update)
+
     def _init_client(
         self,
         url: str,
diff --git a/vault_cli/exceptions.py b/vault_cli/exceptions.py
index 1e92cbd..8300447 100644
--- a/vault_cli/exceptions.py
+++ b/vault_cli/exceptions.py
@@ -18,12 +18,16 @@ class VaultSettingsError(VaultException):
 
 
 class VaultOverwriteSecretError(VaultException):
-    def __init__(self, path: str):
+    def __init__(self, path: str, keys: Optional[Iterable[str]] = None):
         self.path = path
+        self.keys = keys
         super().__init__()
 
     def __str__(self):
-        return f"VaultOverwriteSecretError: Secret at {self.path} already exists"
+        s = f"Secret already exists at {self.path}"
+        if self.keys:
+            s += f" for key{'s' if len(self.keys) > 1 else ''}: {', '.join(self.keys)}"
+        return s
 
 
 class VaultMixSecretAndFolder(VaultException):
diff --git a/vault_cli/metadata.py b/vault_cli/metadata.py
index 64cba4a..a2edcee 100644
--- a/vault_cli/metadata.py
+++ b/vault_cli/metadata.py
@@ -1,15 +1,15 @@
 import io
 from distutils import dist
-from typing import Mapping
+from typing import Mapping, Optional
 
 import pkg_resources
 
 
-def extract_metadata() -> Mapping[str, str]:
+def extract_metadata() -> Mapping[str, Optional[str]]:
 
     distribution = pkg_resources.get_distribution("vault-cli")
     metadata_str = distribution.get_metadata(distribution.PKG_INFO)
-    metadata_obj = dist.DistributionMetadata()  # type: ignore
+    metadata_obj = dist.DistributionMetadata()
     metadata_obj.read_pkg_file(io.StringIO(metadata_str))
 
     return {
diff --git a/vault_cli/sessions.py b/vault_cli/sessions.py
index 875916b..a31b5de 100644
--- a/vault_cli/sessions.py
+++ b/vault_cli/sessions.py
@@ -1,5 +1,5 @@
 import requests
-import urllib3
+import urllib3  # type: ignore
 
 
 class Session(requests.Session):


Test patch:
diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py
index 406fd8a..ac5f244 100644
--- a/tests/unit/test_cli.py
+++ b/tests/unit/test_cli.py
@@ -283,6 +283,46 @@ def test_set_mix_folders_secrets(cli_runner, vault_with_token):
     assert vault_with_token.db == {"a/b/c": {"value": "d"}}
 
 
+def test_set_all(cli_runner, vault_with_token):
+
+    result = cli_runner.invoke(cli.cli, ["set-all"], input="""a/b: {"c": "d"}""")
+
+    assert result.exit_code == 0
+    assert result.stdout.strip() == "Done"
+    assert vault_with_token.db == {"a/b": {"c": "d"}}
+
+
+def test_set_all_wrong_type(cli_runner, vault_with_token):
+    result = cli_runner.invoke(cli.cli, ["set-all"], input="""[1, 2, 3]""")
+    assert result.exit_code != 0
+    error = "Error: Mapping expected format is a mapping of paths to secret objects"
+    assert result.stdout.strip() == error
+    assert vault_with_token.db == {}
+
+
+def test_set_all_force(cli_runner, vault_with_token):
+    vault_with_token.db = {"a/b": {"c": "d"}}
+    result = cli_runner.invoke(
+        cli.cli, ["set-all", "--no-force"], input="""a/b: {"c": "e"}"""
+    )
+    assert result.exit_code != 0
+    error = (
+        "Error: Secret already exists at a/b for key: c\nUse -f to force overwriting"
+    )
+    assert result.stdout.strip() == error
+    assert vault_with_token.db == {"a/b": {"c": "d"}}
+
+
+def test_set_all_mix(cli_runner, vault_with_token):
+    vault_with_token.db = {"a/b": {"c": "d"}}
+    result = cli_runner.invoke(cli.cli, ["set-all"], input="""a: {"e": "f"}""")
+
+    assert result.exit_code != 0
+    error = "Error: Cannot create a secret at 'a' because it is already a folder containing a/b"
+    assert result.stdout.strip() == error
+    assert vault_with_token.db == {"a/b": {"c": "d"}}
+
+
 def test_delete(cli_runner, vault_with_token):
 
     vault_with_token.db = {"a": {"value": "foo"}, "b": {"value": "bar"}}
diff --git a/tests/unit/test_client_base.py b/tests/unit/test_client_base.py
index a03c86e..481cc16 100644
--- a/tests/unit/test_client_base.py
+++ b/tests/unit/test_client_base.py
@@ -722,3 +722,22 @@ def test_vault_client_base_get_secret_explicit_cache(vault):
         # Value not updated
         assert vault.get_secret("a") == {"value": "b"}
     assert vault.get_secret("a") == {"value": "c"}
+
+
+def test_vault_client_base_set_secrets(vault):
+    secrets = {"a": {"b": "c"}, "d/e/f": {"g": "h"}}
+    with vault:
+        vault.set_secrets(secrets)
+    assert vault.db == secrets
+
+
+def test_vault_client_base_set_secrets_update(vault):
+    vault.db = {"a": {"b": "c"}, "d/e/f": {"i": "j"}}
+    vault.set_secrets({"d/e/f": {"g": "h"}}, update=True)
+    assert vault.db == {"a": {"b": "c"}, "d/e/f": {"g": "h", "i": "j"}}
+
+
+def test_vault_client_base_set_secrets_force(vault):
+    vault.db = {"a": {"b": "c"}, "d/e/f": {"i": "j"}}
+    with pytest.raises(exceptions.VaultOverwriteSecretError):
+        vault.set_secrets({"d/e/f": {"g": "h"}}, force=False)
diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py
index e9196f2..7adf371 100644
--- a/tests/unit/test_exceptions.py
+++ b/tests/unit/test_exceptions.py
@@ -3,15 +3,19 @@ import pytest
 from vault_cli import exceptions
 
 
-def test_vault_overwrite_secret_error():
-    assert (
-        str(exceptions.VaultOverwriteSecretError(path="yay"))
-        == "VaultOverwriteSecretError: Secret at yay already exists"
-    )
-
-
-def test_vault_render_template_error():
-    assert str(exceptions.VaultRenderTemplateError("yay")) == "yay"
+@pytest.mark.parametrize(
+    "kwargs, out",
+    [
+        ({"path": "yay"}, "Secret already exists at yay"),
+        ({"path": "yay", "keys": ["a"]}, "Secret already exists at yay for key: a"),
+        (
+            {"path": "yay", "keys": ["a", "b"]},
+            "Secret already exists at yay for keys: a, b",
+        ),
+    ],
+)
+def test_vault_overwrite_secret_error(kwargs, out):
+    assert str(exceptions.VaultOverwriteSecretError(**kwargs)) == out
 
 
 @pytest.mark.parametrize(