Repository: bridgecrewio/checkov

Issue:
Unable to download Terraform modules from JFrog Artifactory
**Describe the issue**

The `--download-external-modules` option fails to download Terraform modules from JFrog Artifactory due to a malformed URL.

**Example Value**

This Terraform configuration:

```
module "lambda" {
  source = "artifactory.jfrog.example.com/example-terraform-module__example/lambda/aws"
  version = "1.8.133"
}
```

produces these warnings:

```
2023-05-31 05:22:07,799 [MainThread  ] [WARNI]  Module artifactory.jfrog.example.com/example-terraform-module__example/lambda/aws:1.8.133 failed to load via <class 'checkov.terraform.module_loading.loaders.registry_loader.RegistryLoader'>
2023-05-31 05:22:07,799 [MainThread  ] [WARNI]  Unable to load module (artifactory.jfrog.example.com/example-terraform-module__example/lambda/aws:1.8.133): HTTPSConnectionPool(host='artifactory.jfrog.example.comhttps', port=443): Max retries exceeded with url: //artifactory.jfrog.example.com/artifactory/api/terraform/v1/modules//example-terraform-module__example/lambda/aws/versions (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f3ee1315570>: Failed to establish a new connection: [Errno -2] Name or service not known'))
```



Discussion:

Resolving patch:
diff --git a/checkov/common/goget/registry/get_registry.py b/checkov/common/goget/registry/get_registry.py
--- a/checkov/common/goget/registry/get_registry.py
+++ b/checkov/common/goget/registry/get_registry.py
@@ -4,18 +4,20 @@
 
 from checkov.common.goget.base_getter import BaseGetter
 from checkov.common.util.file_utils import extract_tar_archive
+from checkov.common.util.file_utils import extract_zip_archive
 from checkov.common.util.http_utils import DEFAULT_TIMEOUT
 
 
 class RegistryGetter(BaseGetter):
-    def __init__(self, url: str, create_clone_and_result_dirs: bool = False) -> None:
+    def __init__(self, url: str, extension: str, create_clone_and_result_dirs: bool = False) -> None:
         self.logger = logging.getLogger(__name__)
+        self.extension = extension
         self.create_clone_and_res_dirs = create_clone_and_result_dirs
         super().__init__(url)
 
     def do_get(self) -> str:
         # get dest dir
-        download_path = os.path.join(self.temp_dir, 'module_source.tar.gz')
+        download_path = os.path.join(self.temp_dir, f'module_source.{self.extension}')
         # download zip
         dest_path = os.path.dirname(download_path)
         with requests.get(self.url, stream=True, timeout=DEFAULT_TIMEOUT) as r:
@@ -25,7 +27,10 @@ def do_get(self) -> str:
                 for chunk in r.iter_content(chunk_size=8192):
                     f.write(chunk)
         # extract
-        extract_tar_archive(source_path=download_path, dest_path=dest_path)
+        if self.extension == 'zip':
+            extract_zip_archive(source_path=download_path, dest_path=dest_path)
+        else:
+            extract_tar_archive(source_path=download_path, dest_path=dest_path)
         os.remove(download_path)
 
         return dest_path
diff --git a/checkov/common/util/file_utils.py b/checkov/common/util/file_utils.py
--- a/checkov/common/util/file_utils.py
+++ b/checkov/common/util/file_utils.py
@@ -5,6 +5,8 @@
 import io
 import logging
 
+from zipfile import ZipFile
+
 
 def convert_to_unix_path(path: str) -> str:
     return path.replace('\\', '/')
@@ -15,6 +17,11 @@ def extract_tar_archive(source_path: str, dest_path: str) -> None:
         tar.extractall(path=dest_path)  # nosec  # only trusted source
 
 
+def extract_zip_archive(source_path: str, dest_path: str) -> None:
+    with ZipFile(source_path) as zip:
+        zip.extractall(path=dest_path)  # nosec  # only trusted source
+
+
 def compress_file_gzip_base64(input_path: str) -> str:
     try:
         with open(input_path, 'rb') as json_results_file:
diff --git a/checkov/terraform/module_loading/loaders/registry_loader.py b/checkov/terraform/module_loading/loaders/registry_loader.py
--- a/checkov/terraform/module_loading/loaders/registry_loader.py
+++ b/checkov/terraform/module_loading/loaders/registry_loader.py
@@ -7,6 +7,7 @@
 
 import requests
 from requests.exceptions import HTTPError
+from urllib.parse import urljoin
 from urllib.parse import urlparse
 
 from checkov.common.models.consts import TFC_HOST_NAME
@@ -42,7 +43,7 @@ def discover(self, module_params: ModuleParams) -> None:
 
     def _is_matching_loader(self, module_params: ModuleParams) -> bool:
         # https://developer.hashicorp.com/terraform/language/modules/sources#github
-        if module_params.module_source.startswith(("github.com", "bitbucket.org", "git::", "git@github.com")):
+        if module_params.module_source.startswith(("/", "github.com", "bitbucket.org", "git::", "git@github.com")):
             return False
         self._process_inner_registry_module(module_params)
         # determine tf api endpoints
@@ -78,7 +79,7 @@ def _load_module(self, module_params: ModuleParams) -> ModuleContent:
         elif not module_params.tf_modules_endpoint:
             return ModuleContent(dir=None)
 
-        request_download_url = "/".join((module_params.tf_modules_endpoint, module_params.module_source, best_version, "download"))
+        request_download_url = urljoin(module_params.tf_modules_endpoint, "/".join((module_params.module_source, best_version, "download")))
         logging.debug(f"Best version for {module_params.module_source} is {best_version} based on the version constraint {module_params.version}.")
         logging.debug(f"Module download url: {request_download_url}")
         try:
@@ -97,9 +98,10 @@ def _load_module(self, module_params: ModuleParams) -> ModuleContent:
         self.logger.debug(f"X-Terraform-Get: {module_download_url}")
         module_download_url = self._normalize_module_download_url(module_params, module_download_url)
         self.logger.debug(f"Cloning module from normalized url {module_download_url}")
-        if self._is_download_url_archive(module_download_url):
+        archive_extension = self._get_archive_extension(module_download_url)
+        if archive_extension:
             try:
-                registry_getter = RegistryGetter(module_download_url)
+                registry_getter = RegistryGetter(module_download_url, archive_extension)
                 registry_getter.temp_dir = module_params.dest_dir
                 registry_getter.do_get()
                 return_dir = module_params.dest_dir
@@ -193,12 +195,14 @@ def _determine_tf_api_endpoints(self, module_params: ModuleParams) -> None:
                     return None
 
             self.logger.debug(f"Service discovery response: {response.json()}")
-            module_params.tf_modules_endpoint = f"https://{module_params.tf_host_name}{response.json().get('modules.v1')}"
+            module_params.tf_modules_endpoint = self._normalize_module_download_url(module_params, response.json().get('modules.v1'))
         else:
             # use terraform cloud host name and url for the public registry
             module_params.tf_host_name = TFC_HOST_NAME
-            module_params.tf_modules_endpoint = "https://registry.terraform.io/v1/modules"
-        module_params.tf_modules_versions_endpoint = "/".join((module_params.tf_modules_endpoint, module_params.module_source, "versions"))
+            module_params.tf_modules_endpoint = "https://registry.terraform.io/v1/modules/"
+
+        # assume module_params.tf_modules_endpoint ends with a slash as per https://developer.hashicorp.com/terraform/internals/module-registry-protocol#service-discovery
+        module_params.tf_modules_versions_endpoint = urljoin(module_params.tf_modules_endpoint, "/".join((module_params.module_source, "versions")))
 
     def _normalize_module_download_url(self, module_params: ModuleParams, module_download_url: str) -> str:
         if not urlparse(module_download_url).netloc:
@@ -206,17 +210,18 @@ def _normalize_module_download_url(self, module_params: ModuleParams, module_dow
         return module_download_url
 
     @staticmethod
-    def _is_download_url_archive(module_download_url: str) -> bool:
+    def _get_archive_extension(module_download_url: str) -> str | None:
+        module_download_path = urlparse(module_download_url).path
         for extension in MODULE_ARCHIVE_EXTENSIONS:
-            if module_download_url.endswith(extension):
-                return True
+            if module_download_path.endswith(extension):
+                return extension
         query_params_str = urlparse(module_download_url).query
         if query_params_str:
             query_params = query_params_str.split("&")
             for query_param in query_params:
                 if query_param.startswith("archive="):
-                    return True
-        return False
+                    return query_params_str.split("=")[1]
+        return None
 
 
 loader = RegistryLoader()


Test patch:
diff --git a/tests/terraform/module_loading/loaders/test_registry_loader.py b/tests/terraform/module_loading/loaders/test_registry_loader.py
--- a/tests/terraform/module_loading/loaders/test_registry_loader.py
+++ b/tests/terraform/module_loading/loaders/test_registry_loader.py
@@ -40,11 +40,34 @@ def test_determine_tf_api_endpoints_tfc():
 
     # then
     assert module_params.tf_host_name == "app.terraform.io"
-    assert module_params.tf_modules_endpoint == "https://registry.terraform.io/v1/modules"
+    assert module_params.tf_modules_endpoint == "https://registry.terraform.io/v1/modules/"
     assert module_params.tf_modules_versions_endpoint == "https://registry.terraform.io/v1/modules/terraform-aws-modules/example/versions"
 
+@pytest.mark.parametrize(
+    "discovery_response",
+    [
+        ({
+        "modules.v1": "/api/registry/v1/modules/",
+        "providers.v1": "/api/registry/v1/providers/",
+        "state.v2": "/api/v2/",
+        "tfe.v2": "/api/v2/",
+        "tfe.v2.1": "/api/v2/",
+        "tfe.v2.2": "/api/v2/",
+        "versions.v1": "https://checkpoint-api.hashicorp.com/v1/versions/"
+        }),
+        ({
+        "modules.v1": "https://example.registry.com/api/registry/v1/modules/",
+        "providers.v1": "https://example.registry.com/api/registry/v1/providers/",
+        "state.v2": "https://example.registry.com/api/v2/",
+        "tfe.v2": "https://example.registry.com/api/v2/",
+        "tfe.v2.1": "https://example.registry.com/api/v2/",
+        "tfe.v2.2": "https://example.registry.com/api/v2/",
+        "versions.v1": "https://checkpoint-api.hashicorp.com/v1/versions/"
+        }),
+    ]
+)
 @responses.activate
-def test_determine_tf_api_endpoints_tfe():
+def test_determine_tf_api_endpoints_tfe(discovery_response):
     # given
     loader = RegistryLoader()
     module_params = ModuleParams("", "", "example.registry.com/terraform-aws-modules/example", "", "", "")
@@ -53,15 +76,7 @@ def test_determine_tf_api_endpoints_tfe():
     responses.add(
         method=responses.GET,
         url=f"https://{module_params.tf_host_name}/.well-known/terraform.json",
-        json={
-        "modules.v1": "/api/registry/v1/modules/",
-        "providers.v1": "/api/registry/v1/providers/",
-        "state.v2": "/api/v2/",
-        "tfe.v2": "/api/v2/",
-        "tfe.v2.1": "/api/v2/",
-        "tfe.v2.2": "/api/v2/",
-        "versions.v1": "https://checkpoint-api.hashicorp.com/v1/versions/"
-        },
+        json=discovery_response,
         status=200,
     )
 
@@ -72,20 +87,42 @@ def test_determine_tf_api_endpoints_tfe():
     responses.assert_call_count(f"https://{module_params.tf_host_name}/.well-known/terraform.json", 1)
     assert module_params.tf_host_name == "example.registry.com"
     assert module_params.tf_modules_endpoint == "https://example.registry.com/api/registry/v1/modules/"
-    assert module_params.tf_modules_versions_endpoint == "https://example.registry.com/api/registry/v1/modules//terraform-aws-modules/example/versions"
+    assert module_params.tf_modules_versions_endpoint == "https://example.registry.com/api/registry/v1/modules/terraform-aws-modules/example/versions"
+
+@responses.activate
+def test_load_module():
+    # given
+    loader = RegistryLoader()
+    module_params = ModuleParams("", "", "terraform-aws-modules/example", "", "", "")
+    module_params.tf_modules_endpoint = "https://example.registry.com/api/registry/v1/modules/"
+    module_params.best_version = "1.0.0"
+    with mock.patch.dict("os.environ", {"TF_HOST_NAME": "example.registry.com"}):
+        loader.discover(module_params)
+    responses.add(
+        method=responses.GET,
+        url="https://example.registry.com/api/registry/v1/modules/terraform-aws-modules/example/1.0.0/download",
+        status=200,
+    )
+
+    # when
+    loader._load_module(module_params)
+
+    # then
+    responses.assert_call_count("https://example.registry.com/api/registry/v1/modules/terraform-aws-modules/example/1.0.0/download", 1)
 
 @pytest.mark.parametrize(
     "download_url, expected_result",
     [
-        ("https://example.com/download?archive=tgz", True),
-        ("https://example.com/download?archive=zip", True),
-        ("https://example.com/download/module.zip", True),
-        ("https://example.com/download/module/archive", False),
+        ("https://example.com/download?archive=tgz", "tgz"),
+        ("https://example.com/download?archive=zip", "zip"),
+        ("https://example.com/download/module.zip", "zip"),
+        ("https://example.com/download/module.zip?sig=foo", "zip"),
+        ("https://example.com/download/module/archive", None),
     ]
 )
-def test_is_download_url_archive(download_url, expected_result):
-    is_archive = RegistryLoader._is_download_url_archive(download_url)
-    assert is_archive == expected_result
+def test_get_archive_extension(download_url, expected_result):
+    archive_extension = RegistryLoader._get_archive_extension(download_url)
+    assert archive_extension == expected_result
 
 @pytest.mark.parametrize(
     "tf_host_name, module_download_url, expected_result",