Repository: bridgecrewio/checkov

Issue:
Please add support for third-party module registries
**Describe the issue**
If an existing integration is malfunctioning, please describe the current state and 
what you expect to be happening.
For new integrations, please share an example use case this integration will help 
checkov support.

Hello, as mentioned in #2098 this issue is opened to request support for the Terraform Module Registry Protocol to Checkov.


Discussion:
@kartikp10 I am having this issue as we have a private repo for Terraform modules. Any updates on this issue?

Resolving patch:
diff --git a/checkov/terraform/module_loading/loaders/registry_loader.py b/checkov/terraform/module_loading/loaders/registry_loader.py
--- a/checkov/terraform/module_loading/loaders/registry_loader.py
+++ b/checkov/terraform/module_loading/loaders/registry_loader.py
@@ -5,6 +5,7 @@
 
 import requests
 from requests.exceptions import HTTPError
+from urllib.parse import urlparse
 
 from checkov.common.models.consts import TFC_HOST_NAME
 from checkov.common.goget.registry.get_registry import RegistryGetter
@@ -17,80 +18,67 @@
 )
 from checkov.terraform.module_loading.module_params import ModuleParams
 
+# https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http
+MODULE_ARCHIVE_EXTENSIONS = ["zip", "tar.bz2", "tar.gz", "tgz", "tar.xz", "txz"]
+
 
 class RegistryLoader(ModuleLoader):
     modules_versions_cache: Dict[str, List[str]] = {}  # noqa: CCE003  # public data
 
     def __init__(self) -> None:
         super().__init__()
-        self.module_version_url = ""
-        self.best_version = ""
 
     def discover(self, module_params):
-        module_params.REGISTRY_URL_PREFIX = os.getenv("REGISTRY_URL_PREFIX", "https://registry.terraform.io/v1/modules")
-        module_params.token = os.getenv("TFC_TOKEN", "")
+        module_params.tf_host_name = os.getenv("TF_HOST_NAME", TFC_HOST_NAME)
+        module_params.token = os.getenv("TF_REGISTRY_TOKEN", "")
+        tfc_token = os.getenv("TFC_TOKEN")
+        if tfc_token:
+            self.logger.warn("Environment variable TFC_TOKEN will be deprecated in the future. Please use TF_REGISTRY_TOKEN instead.")
+            module_params.token = tfc_token
 
     def _is_matching_loader(self, module_params: ModuleParams) -> bool:
-
-        # Since the registry loader is the first one to be checked,
-        # it shouldn't process any github modules
-        if module_params.module_source.startswith(("github.com", "bitbucket.org", "git::")):
+        # https://developer.hashicorp.com/terraform/language/modules/sources#github
+        if module_params.module_source.startswith(("github.com", "bitbucket.org", "git::", "git@github.com")):
             return False
-
         self._process_inner_registry_module(module_params)
-        if os.path.exists(module_params.dest_dir):
-            return True
-
-        if module_params.module_source.startswith(TFC_HOST_NAME):
-            # indicates a private registry module
-            module_params.REGISTRY_URL_PREFIX = f"https://{TFC_HOST_NAME}/api/registry/v1/modules"
-            module_params.module_source = module_params.module_source.replace(f"{TFC_HOST_NAME}/", "")
-        else:
-            # url for the public registry
-            module_params.REGISTRY_URL_PREFIX = "https://registry.terraform.io/v1/modules"
-
-        if module_params.module_source.startswith(module_params.REGISTRY_URL_PREFIX):
-            # TODO: implement registry url validation using remote service discovery
-            # https://www.terraform.io/internals/remote-service-discovery#remote-service-discovery
-            pass
-        module_params.module_version_url = "/".join((module_params.REGISTRY_URL_PREFIX, module_params.module_source, "versions"))
-        if not module_params.module_version_url.startswith(module_params.REGISTRY_URL_PREFIX):
-            # Local paths don't get the prefix appended
-            return False
-
+        # determine tf api endpoints
+        self._determine_tf_api_endpoints(module_params)
         # If versions for a module are cached, determine the best version and return True.
         # If versions are not cached, get versions, then determine the best version and return True.
         # Best version needs to be determined here for setting most accurate dest_dir.
-        if module_params.module_version_url in RegistryLoader.modules_versions_cache.keys():
+        if module_params.tf_modules_versions_endpoint in RegistryLoader.modules_versions_cache.keys():
             module_params.best_version = self._find_best_version(module_params)
             return True
         if not self._cache_available_versions(module_params):
             return False
         module_params.best_version = self._find_best_version(module_params)
-
         if not module_params.inner_module:
             module_params.dest_dir = os.path.join(module_params.root_dir, module_params.external_modules_folder_name,
-                                                  TFC_HOST_NAME, *module_params.module_source.split("/"),
+                                                  module_params.tf_host_name, *module_params.module_source.split("/"),
                                                   module_params.best_version)
         if os.path.exists(module_params.dest_dir):
             return True
         # verify cache again after refresh
-        if module_params.module_version_url in RegistryLoader.modules_versions_cache.keys():
+        if module_params.tf_modules_versions_endpoint in RegistryLoader.modules_versions_cache.keys():
             return True
         return False
 
     def _load_module(self, module_params: ModuleParams) -> ModuleContent:
+        if module_params.best_version:
+            best_version = module_params.best_version
+        else:
+            if self._cache_available_versions(module_params):
+                module_params.best_version = self._find_best_version(module_params)
         if os.path.exists(module_params.dest_dir):
             return ModuleContent(dir=module_params.dest_dir)
 
-        best_version = module_params.best_version
-        logging.debug(
-            f"Best version for {module_params.module_source} is {best_version} based on the version constraint {module_params.version}")
-        request_download_url = "/".join((module_params.REGISTRY_URL_PREFIX, module_params.module_source, best_version, "download"))
+        request_download_url = "/".join((module_params.tf_modules_endpoint, module_params.module_source, best_version, "download"))
+        logging.debug(f"Best version for {module_params.module_source} is {best_version} based on the version constraint {module_params.version}.")
+        logging.debug(f"Module download url: {request_download_url}")
         try:
             response = requests.get(
                 url=request_download_url,
-                headers={"Authorization": f"Bearer {module_params.token}"},
+                headers={"Authorization": f"Bearer {module_params.token}"} if module_params.token else None,
                 timeout=DEFAULT_TIMEOUT
             )
             response.raise_for_status()
@@ -98,33 +86,34 @@ def _load_module(self, module_params: ModuleParams) -> ModuleContent:
             self.logger.warning(e)
             if response.status_code != HTTPStatus.OK and response.status_code != HTTPStatus.NO_CONTENT:
                 return ModuleContent(dir=None)
+        # https://www.terraform.io/registry/api-docs#download-source-code-for-a-specific-module-version
+        module_download_url = response.headers.get('X-Terraform-Get', '')
+        self.logger.debug(f"X-Terraform-Get: {module_download_url}")
+        module_download_url = self._normalize_module_download_url(module_params, module_download_url)
+        self.logger.debug(f"Cloning module from normalized url {module_download_url}")
+        if self._is_download_url_archive(module_download_url):
+            try:
+                registry_getter = RegistryGetter(module_download_url)
+                registry_getter.temp_dir = module_params.dest_dir
+                registry_getter.do_get()
+                return_dir = module_params.dest_dir
+            except Exception as e:
+                str_e = str(e)
+                if 'File exists' not in str_e and 'already exists and is not an empty directory' not in str_e:
+                    self.logger.error(f"failed to get {module_params.module_source} because of {e}")
+                    return ModuleContent(dir=None, failed_url=module_params.module_source)
+            if module_params.inner_module:
+                return_dir = os.path.join(module_params.dest_dir, module_params.inner_module)
+            return ModuleContent(dir=return_dir)
         else:
-            # https://www.terraform.io/registry/api-docs#download-source-code-for-a-specific-module-version
-            module_download_url = response.headers.get('X-Terraform-Get', '')
-            self.logger.debug(f"Cloning module from: X-Terraform-Get: {module_download_url}")
-            if module_download_url.startswith("https://archivist.terraform.io/v1/object"):
-                try:
-                    registry_getter = RegistryGetter(module_download_url)
-                    registry_getter.temp_dir = module_params.dest_dir
-                    registry_getter.do_get()
-                    return_dir = module_params.dest_dir
-                except Exception as e:
-                    str_e = str(e)
-                    if 'File exists' not in str_e and 'already exists and is not an empty directory' not in str_e:
-                        self.logger.error(f"failed to get {module_params.module_source} because of {e}")
-                        return ModuleContent(dir=None, failed_url=module_params.module_source)
-                if module_params.inner_module:
-                    return_dir = os.path.join(module_params.dest_dir, module_params.inner_module)
-                return ModuleContent(dir=return_dir)
-            else:
-                return ModuleContent(dir=None, next_url=response.headers.get("X-Terraform-Get", ""))
+            return ModuleContent(dir=None, next_url=response.headers.get("X-Terraform-Get", ""))
 
     def _find_module_path(self, module_params: ModuleParams) -> str:
         # to determine the exact path here would be almost a duplicate of the git_loader functionality
         return ""
 
     def _find_best_version(self, module_params: ModuleParams) -> str:
-        versions_by_size = RegistryLoader.modules_versions_cache.get(module_params.module_version_url, [])
+        versions_by_size = RegistryLoader.modules_versions_cache.get(module_params.tf_modules_versions_endpoint, [])
         if module_params.version == "latest":
             module_params.version = versions_by_size[0]
         version_constraints = get_version_constraints(module_params.version)
@@ -146,15 +135,15 @@ def _cache_available_versions(self, module_params: ModuleParams) -> bool:
         # Returns False on failure.
         try:
             response = requests.get(
-                url=module_params.module_version_url,
-                headers={"Authorization": f"Bearer {module_params.token}"},
+                url=module_params.tf_modules_versions_endpoint,
+                headers={"Authorization": f"Bearer {module_params.token}"} if module_params.token else None,
                 timeout=DEFAULT_TIMEOUT,
             )
             response.raise_for_status()
             available_versions = [
                 v.get("version") for v in response.json().get("modules", [{}])[0].get("versions", {})
             ]
-            RegistryLoader.modules_versions_cache[module_params.module_version_url] = order_versions_in_descending_order(
+            RegistryLoader.modules_versions_cache[module_params.tf_modules_versions_endpoint] = order_versions_in_descending_order(
                 available_versions)
             return True
         except HTTPError as e:
@@ -172,5 +161,50 @@ def _process_inner_registry_module(self, module_params: ModuleParams) -> None:
             module_params.dest_dir = module_params.dest_dir.split("//")[0]
             module_params.inner_module = module_source_components[1]
 
+    def _determine_tf_api_endpoints(self, module_params: ModuleParams) -> None:
+        """
+        Determines terraform registry endpoints - tf_host_name, tf_modules_endpoint, tf_modules_versions_endpoint
+        """
+        if module_params.module_source.startswith(module_params.tf_host_name):
+            # check if module source supports native Terraform services
+            # https://www.terraform.io/internals/remote-service-discovery#remote-service-discovery
+            module_params.module_source = module_params.module_source.replace(f"{module_params.tf_host_name}/", "")
+            try:
+                response = requests.get(
+                    url=f"https://{module_params.tf_host_name}/.well-known/terraform.json",
+                    timeout=DEFAULT_TIMEOUT,
+                )
+                response.raise_for_status()
+            except HTTPError as e:
+                self.logger.debug(e)
+                if response.status_code != HTTPStatus.OK and response.status_code != HTTPStatus.NO_CONTENT:
+                    return False
+
+            self.logger.debug(f"Service discovery response: {response.json()}")
+            module_params.tf_modules_endpoint = f"https://{module_params.tf_host_name}{response.json().get('modules.v1')}"
+        else:
+            # use terraform cloud host name and url for the public registry
+            module_params.tf_host_name = TFC_HOST_NAME
+            module_params.tf_modules_endpoint = "https://registry.terraform.io/v1/modules"
+        module_params.tf_modules_versions_endpoint = "/".join((module_params.tf_modules_endpoint, module_params.module_source, "versions"))
+
+    def _normalize_module_download_url(self, module_params: ModuleParams, module_download_url) -> str:
+        if not urlparse(module_download_url).netloc:
+            module_download_url = f"https://{module_params.tf_host_name}{module_download_url}"
+        return module_download_url
+
+    @staticmethod
+    def _is_download_url_archive(module_download_url) -> bool:
+        for extension in MODULE_ARCHIVE_EXTENSIONS:
+            if module_download_url.endswith(extension):
+                return True
+        query_params = urlparse(module_download_url).query
+        if query_params:
+            query_params = query_params.split("&")
+            for query_param in query_params:
+                if query_param.startswith("archive="):
+                    return True
+        return False
+
 
 loader = RegistryLoader()
diff --git a/checkov/terraform/module_loading/module_params.py b/checkov/terraform/module_loading/module_params.py
--- a/checkov/terraform/module_loading/module_params.py
+++ b/checkov/terraform/module_loading/module_params.py
@@ -18,6 +18,9 @@ def __init__(self, root_dir: str, current_dir: str, source: str, source_version:
         self.username: Optional[str] = None
         self.vcs_base_url: Optional[str] = None
         self.module_source_prefix: Optional[str] = None
-        self.REGISTRY_URL_PREFIX: Optional[str] = None
         self.best_version: Optional[str] = None
-        self.module_version_url: Optional[str] = None
+
+        # terraform cloud / enterprise specific params
+        self.tf_host_name: Optional[str] = None
+        self.tf_modules_endpoint: Optional[str] = None
+        self.tf_modules_versions_endpoint: Optional[str] = None


Test patch:
diff --git a/integration_tests/prepare_data.sh b/integration_tests/prepare_data.sh
--- a/integration_tests/prepare_data.sh
+++ b/integration_tests/prepare_data.sh
@@ -39,6 +39,6 @@ then
   echo "running list"
   pipenv run checkov --list --bc-api-key $BC_KEY --output-bc-ids > checkov_checks_list.txt
   echo "running tfc"
-#  GITHUB_PAT="$GITHUB_PAT" TFC_TOKEN="$TFC_TOKEN" pipenv run checkov -s -d integration_tests/example_ext_private_modules/ --download-external-modules True
+#  GITHUB_PAT="$GITHUB_PAT" TF_REGISTRY_TOKEN="$TFC_TOKEN" pipenv run checkov -s -d integration_tests/example_ext_private_modules/ --download-external-modules True
 
 fi
diff --git a/tests/terraform/module_loading/loaders/test_registry_loader.py b/tests/terraform/module_loading/loaders/test_registry_loader.py
--- a/tests/terraform/module_loading/loaders/test_registry_loader.py
+++ b/tests/terraform/module_loading/loaders/test_registry_loader.py
@@ -1,4 +1,6 @@
 import responses
+import pytest
+from unittest import mock
 
 from checkov.terraform.module_loading.loaders.registry_loader import RegistryLoader
 from checkov.terraform.module_loading.module_params import ModuleParams
@@ -17,6 +19,7 @@ def test_module_version_url_invoked_once():
     loader = RegistryLoader()
     RegistryLoader.modules_versions_cache = {}  # reset cache
     module_params = ModuleParams("", "", "terraform-aws-modules/example", "", "", "")
+    loader.discover(module_params)
 
     # when
     loader._is_matching_loader(module_params)
@@ -25,3 +28,100 @@ def test_module_version_url_invoked_once():
     # then
     responses.assert_call_count(module_version_url, 1)
     assert loader.modules_versions_cache == {module_version_url: ["1.0.0"]}
+
+def test_determine_tf_api_endpoints_tfc():
+    # given
+    loader = RegistryLoader()
+    module_params = ModuleParams("", "", "terraform-aws-modules/example", "", "", "")
+    loader.discover(module_params)
+
+    # when
+    loader._determine_tf_api_endpoints(module_params)
+
+    # then
+    assert module_params.tf_host_name == "app.terraform.io"
+    assert module_params.tf_modules_endpoint == "https://registry.terraform.io/v1/modules"
+    assert module_params.tf_modules_versions_endpoint == "https://registry.terraform.io/v1/modules/terraform-aws-modules/example/versions"
+
+@responses.activate
+def test_determine_tf_api_endpoints_tfe():
+    # given
+    loader = RegistryLoader()
+    module_params = ModuleParams("", "", "example.registry.com/terraform-aws-modules/example", "", "", "")
+    with mock.patch.dict("os.environ", {"TF_HOST_NAME": "example.registry.com"}):
+        loader.discover(module_params)
+    responses.add(
+        method=responses.GET,
+        url=f"https://{module_params.tf_host_name}/.well-known/terraform.json",
+        json={
+        "modules.v1": "/api/registry/v1/modules/",
+        "providers.v1": "/api/registry/v1/providers/",
+        "state.v2": "/api/v2/",
+        "tfe.v2": "/api/v2/",
+        "tfe.v2.1": "/api/v2/",
+        "tfe.v2.2": "/api/v2/",
+        "versions.v1": "https://checkpoint-api.hashicorp.com/v1/versions/"
+        },
+        status=200,
+    )
+
+    # when
+    loader._determine_tf_api_endpoints(module_params)
+
+    # then
+    responses.assert_call_count(f"https://{module_params.tf_host_name}/.well-known/terraform.json", 1)
+    assert module_params.tf_host_name == "example.registry.com"
+    assert module_params.tf_modules_endpoint == "https://example.registry.com/api/registry/v1/modules/"
+    assert module_params.tf_modules_versions_endpoint == "https://example.registry.com/api/registry/v1/modules//terraform-aws-modules/example/versions"
+
+@pytest.mark.parametrize(
+    "download_url, expected_result",
+    [
+        ("https://example.com/download?archive=tgz", True),
+        ("https://example.com/download?archive=zip", True),
+        ("https://example.com/download/module.zip", True),
+        ("https://example.com/download/module/archive", False),
+    ]
+)
+def test_is_download_url_archive(download_url, expected_result):
+    is_archive = RegistryLoader._is_download_url_archive(download_url)
+    assert is_archive == expected_result
+
+@pytest.mark.parametrize(
+    "tf_host_name, module_download_url, expected_result",
+    [
+        ("example.com", "https://example.com/download?archive=tgz", "https://example.com/download?archive=tgz"),
+        ("example.com", "https://example.com/abc", "https://example.com/abc"),
+        ("example.com", "/api/registry/v1/modules/namespace/version/download?archive=tgz", "https://example.com/api/registry/v1/modules/namespace/version/download?archive=tgz"),
+    ]
+)
+def test_normalize_module_download_url(tf_host_name, module_download_url, expected_result):
+    # given
+    loader = RegistryLoader()
+    module_params = ModuleParams("", "", "example.com/terraform-aws-modules/example", "", "", "")
+    with mock.patch.dict("os.environ", {"TF_HOST_NAME": tf_host_name}):
+        loader.discover(module_params)
+
+    # when
+    normalized_url = loader._normalize_module_download_url(module_params, module_download_url)
+
+    # then
+    assert normalized_url == expected_result
+
+@pytest.mark.parametrize(
+    "source_url",
+    [
+        ("git::https://example.com/repo.git"),
+        ("git@github.com:org/repo"),
+        ("github.com/org/repo"),
+        ("bitbucket.org/org/repo"),
+    ]
+)
+def test_is_matching_loader_git_sources(source_url):
+    #given
+    loader = RegistryLoader()
+    module_params = ModuleParams("", "", source_url, "", "", "")
+    loader.discover(module_params)
+
+    # then
+    assert not loader._is_matching_loader(module_params)