Repository: gitlabform/gitlabform
Issue: Setting deploy key in some cases fail with HTTP 400: {"fingerprint":["has already been taken"]}

For further investigation:

```
* [215/420] Processing: internal_apps/watchdog
+++ Error while processing 'internal_apps/watchdog'
Traceback (most recent call last):
  File "/srv/gitlab_tools/venv/lib64/python3.5/site-packages/gitlabform/gitlabform/core.py", line 211, in process_all
    self.process_deploy_keys(project_and_group, configuration)
  File "/srv/gitlab_tools/venv/lib64/python3.5/site-packages/gitlabform/gitlabform/core.py", line 31, in method_wrapper
    return method(self, project_and_group, configuration)
  File "/srv/gitlab_tools/venv/lib64/python3.5/site-packages/gitlabform/gitlabform/core.py", line 235, in process_deploy_keys
    self.gl.post_deploy_key(project_and_group, configuration['deploy_keys'][deploy_key])
  File "/srv/gitlab_tools/venv/lib64/python3.5/site-packages/gitlabform/gitlab/projects.py", line 40, in post_deploy_key
    self._make_requests_to_api("projects/%s/deploy_keys", pid, 'POST', deploy_key, expected_codes=201)
  File "/srv/gitlab_tools/venv/lib64/python3.5/site-packages/gitlabform/gitlab/core.py", line 47, in _make_requests_to_api
    response = self._make_request_to_api(path_as_format_string, args, method, data, expected_codes)
  File "/srv/gitlab_tools/venv/lib64/python3.5/site-packages/gitlabform/gitlab/core.py", line 83, in _make_request_to_api
    raise e
gitlabform.gitlab.core.UnexpectedResponseException: Request url='https://xxx/api/v3/projects/1144/deploy_keys', method=POST, data='{'key': 'ssh-ed25519 yyyyyyyyyyy build@xxx.com', 'can_push': True, 'title': 'build@xxx.com auto-bump key'}' failed - expected code(s) 201, got code 400 & body: 'b'{"message":{"fingerprint":["has already been taken"]}}''
```

Changed files:
tests/acceptance/standard/test_deploy_keys_all_projects.py, tests/acceptance/standard/test_deploy_keys.py, gitlabform/gitlab/project_deploy_keys.py

diff --git a/gitlabform/gitlab/project_deploy_keys.py b/gitlabform/gitlab/project_deploy_keys.py
index f5a5d92..34eb6d5 100644
--- a/gitlabform/gitlab/project_deploy_keys.py
+++ b/gitlabform/gitlab/project_deploy_keys.py
@@ -1,13 +1,18 @@
+from gitlabform.gitlab.core import UnexpectedResponseException
 from gitlabform.gitlab.projects import GitLabProjects
 
 
 class GitLabProjectDeployKeys(GitLabProjects):
+    def get_all_deploy_keys(self):
+        return self._make_requests_to_api("deploy_keys")
+
     def get_deploy_keys(self, project_and_group_name):
         return self._make_requests_to_api(
             "projects/%s/deploy_keys", project_and_group_name
         )
 
     def post_deploy_key(self, project_and_group_name, deploy_key_in_config):
+
         # deploy_key_in_config has to be like this:
         # {
         #     'title': title,
@@ -15,13 +20,54 @@ class GitLabProjectDeployKeys(GitLabProjects):
         #     'can_push': can_push,
         # }
         # ..as documented at: https://docs.gitlab.com/ce/api/deploy_keys.html#add-deploy-key
-        self._make_requests_to_api(
-            "projects/%s/deploy_keys",
-            project_and_group_name,
-            "POST",
-            deploy_key_in_config,
-            expected_codes=201,
-        )
+
+        try:
+            self._make_requests_to_api(
+                "projects/%s/deploy_keys",
+                project_and_group_name,
+                "POST",
+                deploy_key_in_config,
+                expected_codes=[201],
+            )
+
+        except UnexpectedResponseException as e:
+            if e.response_status_code == 400 and (
+                "has already been taken" in e.response_text
+            ):
+
+                # Sometimes GitLab throws HTTP 400: {"deploy_key.fingerprint_sha256":["has already been taken"]}
+                # when you try to add an existing SSH key to another project, although according to the API docs
+                # it should work
+                # (see https://docs.gitlab.com/ee/api/deploy_keys.html#add-deploy-keys-to-multiple-projects).
+
+                # As a workaround, when that happens, we will try to get this key's id and enable this existing key
+                # for the new project.
+
+                all_existing_keys = self.get_all_deploy_keys()
+
+                existing_key_id = None
+                for existing_key in all_existing_keys:
+                    if self._keys_are_effectively_equal(
+                        existing_key["key"], deploy_key_in_config["key"]
+                    ):
+                        existing_key_id = existing_key["id"]
+                        break
+
+                if existing_key_id:
+                    # POST /projects/:id/deploy_keys/:key_id/enable
+                    self._make_requests_to_api(
+                        "projects/%s/deploy_keys/%s/enable",
+                        (project_and_group_name, existing_key_id),
+                        "POST",
+                        expected_codes=201,
+                    )
+                else:
+                    e.message = (
+                        e.message
+                        + "\\n"
+                        + "!!! However we haven't found this key in GitLab under the /deploy_keys API... !!!"
+                    )
+                    raise e
 
     def put_deploy_key(
         self, project_and_group_name, deploy_key_in_gitlab, deploy_key_in_config
@@ -53,3 +99,16 @@ class GitLabProjectDeployKeys(GitLabProjects):
         return self._make_requests_to_api(
             "projects/%s/deploy_keys/%s", (project_and_group_name, id), "GET"
         )
+
+    @staticmethod
+    def _keys_are_effectively_equal(key1, key2):
+        # We ignore the comment part of the SSH key as GitLab doesn't allow adding the same key just
+        # with a different comment BUT it also has a bug that it returns keys with only parts of the
+        # comments if the comment contains spaces, so it may show a difference where there is none...
+
+        key1_type = key1.split(" ")[0]
+        key1_value = key1.split(" ")[1]
+        key2_type = key2.split(" ")[0]
+        key2_value = key2.split(" ")[1]
+
+        return (key1_type == key2_type) and (key1_value == key2_value)
diff --git a/tests/acceptance/standard/test_deploy_keys.py b/tests/acceptance/standard/test_deploy_keys.py
index c557479..7ec54d0 100755
--- a/tests/acceptance/standard/test_deploy_keys.py
+++ b/tests/acceptance/standard/test_deploy_keys.py
@@ -59,6 +59,35 @@ class TestDeployKeys:
         deploy_keys = gitlab.get_deploy_keys(group_and_project_for_function)
         assert not any([key["title"] == "some_key" for key in deploy_keys])
 
+    def test__deploy_key_add_delete_with_enforce(
+        self, gitlab, group_and_project_for_function, public_ssh_key
+    ):
+
+        config_add = f"""
+        projects_and_groups:
+          {group_and_project_for_function}:
+            deploy_keys:
+              foobar:
+                key: {public_ssh_key}
+                title: some_key
+        """
+        run_gitlabform(config_add, group_and_project_for_function)
+
+        deploy_keys = gitlab.get_deploy_keys(group_and_project_for_function)
+        assert len(deploy_keys) >= 1
+        assert single_true([key["title"] == "some_key" for key in deploy_keys])
+
+        config_delete = f"""
+        projects_and_groups:
+          {group_and_project_for_function}:
+            deploy_keys:
+              enforce: true
+        """
+        run_gitlabform(config_delete, group_and_project_for_function)
+
+        deploy_keys = gitlab.get_deploy_keys(group_and_project_for_function)
+        assert len(deploy_keys) == 0
+
     def test__deploy_key_add_delete_readd(
         self, gitlab, group_and_project_for_function, public_ssh_key
     ):
diff --git a/tests/acceptance/standard/test_deploy_keys_all_projects.py b/tests/acceptance/standard/test_deploy_keys_all_projects.py
index c8ba28c..66aefba 100755
--- a/tests/acceptance/standard/test_deploy_keys_all_projects.py
+++ b/tests/acceptance/standard/test_deploy_keys_all_projects.py
@@ -23,3 +23,25 @@ class TestDeployKeysAllProjects:
 
         deploy_keys2 = gitlab.get_deploy_keys(f"{group}/{other_project}")
         assert len(deploy_keys2) == 1
+
+    def test__deploy_key_with_spaces_in_comment_to_all_projects(
+        self, gitlab, group, project, other_project, public_ssh_key
+    ):
+        public_ssh_key = f"{public_ssh_key} this is a comment with spaces"
+
+        deploy_key_to_all_projects = f"""
+        projects_and_groups:
+          "*":
+            deploy_keys:
+              foobar:
+                key: {public_ssh_key}
+                title: common_key
+                can_push: false
+        """
+        run_gitlabform(deploy_key_to_all_projects, group)
+
+        deploy_keys1 = gitlab.get_deploy_keys(f"{group}/{project}")
+        assert len(deploy_keys1) == 1
+
+        deploy_keys2 = gitlab.get_deploy_keys(f"{group}/{other_project}")
+        assert len(deploy_keys2) == 1