Repository: packit/packit
Issue: Can packit Centos 9 Streams repositories

When trying to run packit for Centos 9 Streams cloud-init got following error:

2021-03-22 11:49:18.734 utils.py          ERROR  Dist-git URL git@gitlab.com:redhat/centos-stream/rpms/cloud-init.git not recognized, we expected centos.org or fedoraproject.org

Used command is (run in checkouted git@gitlab.com:redhat/centos-stream/rpms/cloud-init.git directory)

packit init --upstream-url git://git.launchpad.net/cloud-init --upstream-ref 47f4229ebcef9f83df8b549bb869a2dbf6dff17c --dist-git-path /tmp/cloud-init --dist-git-branch c9s

Changed files:
packit/exceptions.py, packit/cli/init.py, packit/constants.py, tests/unit/test_source_git_generator.py, packit/cli/utils.py, packit/utils/repo.py, tests/integration/test_source_git_generator.py, packit/source_git.py

diff --git a/packit/cli/init.py b/packit/cli/init.py
index fc0507f0..f5b7d58d 100644
--- a/packit/cli/init.py
+++ b/packit/cli/init.py
@@ -36,7 +36,7 @@ from packit.cli.utils import cover_packit_exception, get_packit_api
 from packit.config import get_context_settings
 from packit.config.config import pass_config
 from packit.constants import CONFIG_FILE_NAMES, PACKIT_CONFIG_TEMPLATE
-from packit.exceptions import PackitException
+from packit.exceptions import PackitException, PackitNotAGitRepoException
 
 logger = logging.getLogger(__name__)
 
@@ -69,7 +69,7 @@ logger = logging.getLogger(__name__)
 @click.option(
     "--dist-git-branch",
     help="Get spec file from this downstream branch, "
-    "for Fedora this defaults to main, for CentOS it's c8s. "
+    "for Fedora this defaults to main, for CentOS it's c9s. "
     "When --dist-git-path is set, the default is the branch which is already checked out.",
 )
 @click.option(
@@ -117,9 +117,18 @@ def init(
             "Generating source-git repositories is experimental, "
             "please give us feedback if it does things differently than you expect."
         )
-        api = get_packit_api(
-            config=config, local_project=path_or_url, load_packit_yaml=False
-        )
+        try:
+            api = get_packit_api(
+                config=config, local_project=path_or_url, load_packit_yaml=False
+            )
+        except PackitNotAGitRepoException:
+            logger.error(
+                "The init command is expected to be run in a git repository. "
+                "Current branch in the repo will be turned into a source-git repo. "
+                "We suggest to run the command "
+                "in a blank git repository or in a new branch of the upstream project."
+            )
+            raise
         dg_path = Path(dist_git_path) if dist_git_path else None
         api.create_sourcegit_from_upstream(
             upstream_url=upstream_url,
diff --git a/packit/cli/utils.py b/packit/cli/utils.py
index aae084b7..6dc1f34a 100644
--- a/packit/cli/utils.py
+++ b/packit/cli/utils.py
@@ -35,7 +35,7 @@ from packit.config.package_config import PackageConfig
 from packit.api import PackitAPI
 from packit.config import Config, get_local_package_config
 from packit.constants import DIST_GIT_HOSTNAME_CANDIDATES
-from packit.exceptions import PackitException
+from packit.exceptions import PackitException, PackitNotAGitRepoException
 from packit.local_project import LocalProject
 
 logger = logging.getLogger(__name__)
@@ -135,7 +135,9 @@ def get_packit_api(
         )
 
     if not local_project.git_repo:
-        raise PackitException(f"{local_project.working_dir!r} is not a git repository.")
+        raise PackitNotAGitRepoException(
+            f"{local_project.working_dir!r} is not a git repository."
+        )
 
     remote_urls: List[str] = []
     for remote in local_project.git_repo.remotes:
diff --git a/packit/constants.py b/packit/constants.py
index 9e297c5f..7279ef88 100644
--- a/packit/constants.py
+++ b/packit/constants.py
@@ -38,12 +38,16 @@ DEFAULT_ARCHIVE_EXT = ".tar.gz"
 
 DEFAULT_BODHI_NOTE = "New upstream release: {version}"
 
-PROD_DISTGIT_HOSTNAME = "src.fedoraproject.org"
+FEDORA_DOMAIN = "fedoraproject.org"
+CENTOS_DOMAIN = "centos.org"
+CENTOS_STREAM_GITLAB = "gitlab.com/redhat/centos-stream"
+
+PROD_DISTGIT_HOSTNAME = f"src.{FEDORA_DOMAIN}"
 PROD_DISTGIT_URL = f"https://{PROD_DISTGIT_HOSTNAME}/"
 
-ALTERNATIVE_PROD_DG_HOSTNAME = "pkgs.fedoraproject.org"
-STG_DISTGIT_HOSTNAME = "src.stg.fedoraproject.org"
-ALTERNATIVE_STG_DG_HOSTNAME = "pkgs.stg.fedoraproject.org"
+ALTERNATIVE_PROD_DG_HOSTNAME = f"pkgs.{FEDORA_DOMAIN}"
+STG_DISTGIT_HOSTNAME = f"src.stg.{FEDORA_DOMAIN}"
+ALTERNATIVE_STG_DG_HOSTNAME = f"pkgs.stg.{FEDORA_DOMAIN}"
 
 DIST_GIT_HOSTNAME_CANDIDATES = (
     PROD_DISTGIT_HOSTNAME,
diff --git a/packit/exceptions.py b/packit/exceptions.py
index d8e8885c..39dbf413 100644
--- a/packit/exceptions.py
+++ b/packit/exceptions.py
@@ -112,3 +112,7 @@ class PackitFailedToCreateRPMException(PackitRPMException):
 
 class PackitGitException(PackitException):
     """ Operation with a git repo failed """
+
+
+class PackitNotAGitRepoException(PackitGitException):
+    """ Target directory is not a git repository as we expected """
diff --git a/packit/source_git.py b/packit/source_git.py
index 3f131279..047a9dbc 100644
--- a/packit/source_git.py
+++ b/packit/source_git.py
@@ -16,19 +16,26 @@ from git import GitCommandError
 from packit.config.common_package_config import CommonPackageConfig
 from packit.config.config import Config
 from packit.config.package_config import PackageConfig
-from packit.constants import RPM_MACROS_FOR_PREP
+from packit.constants import (
+    RPM_MACROS_FOR_PREP,
+    FEDORA_DOMAIN,
+    CENTOS_DOMAIN,
+    CENTOS_STREAM_GITLAB,
+)
 from packit.distgit import DistGit
 from packit.exceptions import PackitException
 from packit.local_project import LocalProject
 from packit.utils.commands import run_command
-from packit.utils.repo import clone_centos_package
+from packit.utils.repo import clone_centos_8_package, clone_centos_9_package
 
 logger = logging.getLogger(__name__)
 
 
-# TODO: we'll need to distinct b/w 8 and 9 - 9 will follow Fedora
-class CentOSDistGit(DistGit):
-    """ CentOS dist-git layout implementation """
+class CentOS8DistGit(DistGit):
+    """
+    CentOS dist-git layout implementation for 8: CentOS Linux 8 and CentOS Stream 8
+    which lives in git.centos.org
+    """
 
     # spec files are stored in this dir in dist-git
     spec_dir_name = "SPECS"
@@ -44,8 +51,35 @@ class CentOSDistGit(DistGit):
         package_config: CommonPackageConfig,
         path: Path,
         branch: str = None,
-    ) -> "CentOSDistGit":
-        clone_centos_package(
+    ) -> "CentOS8DistGit":
+        clone_centos_8_package(
+            package_config.downstream_package_name, path, branch=branch
+        )
+        lp = LocalProject(working_dir=path)
+        return cls(config, package_config, local_project=lp)
+
+
+class CentOS9DistGit(DistGit):
+    """
+    CentOS dist-git layout implementation for CentOS Stream 9
+    which lives in gitlab.com/redhat/centos-stream/rpms
+    """
+
+    # spec files are stored in this dir in dist-git
+    spec_dir_name = ""
+
+    # sources are stored in this dir in dist-git
+    source_dir_name = ""
+
+    @classmethod
+    def clone(
+        cls,
+        config: Config,
+        package_config: CommonPackageConfig,
+        path: Path,
+        branch: str = None,
+    ) -> "CentOS9DistGit":
+        clone_centos_9_package(
             package_config.downstream_package_name, path, branch=branch
         )
         lp = LocalProject(working_dir=path)
@@ -61,12 +95,15 @@ def get_distgit_kls_from_repo(
     path = Path(repo_path)
     pc = PackageConfig(downstream_package_name=path.name)
     lp = LocalProject(working_dir=path)
-    if "fedoraproject.org" in lp.git_url:
+    if FEDORA_DOMAIN in lp.git_url:
         return DistGit(config, pc, local_project=lp), None, path.name
-    elif "centos.org" in lp.git_url:
-        return CentOSDistGit(config, pc, local_project=lp), path.name, None
+    elif CENTOS_DOMAIN in lp.git_url:
+        return CentOS8DistGit(config, pc, local_project=lp), path.name, None
+    elif CENTOS_STREAM_GITLAB in lp.git_url:
+        return CentOS9DistGit(config, pc, local_project=lp), path.name, None
     raise PackitException(
-        f"Dist-git URL {lp.git_url} not recognized, we expected centos.org or fedoraproject.org"
+        f"Dist-git URL {lp.git_url} not recognized, we expected one of: "
+        f"{FEDORA_DOMAIN}, {CENTOS_DOMAIN} or {CENTOS_STREAM_GITLAB}"
     )
 
 
@@ -125,6 +162,10 @@ class SourceGitGenerator:
         self._upstream_ref: Optional[str] = upstream_ref
         self.dist_git_branch = dist_git_branch
 
+        logger.info(
+            f"The source-git repo is going to be created in {local_project.working_dir}."
+        )
+
         if dist_git_path:
             (
                 self._dist_git,
@@ -140,13 +181,17 @@ class SourceGitGenerator:
                 self.package_config = PackageConfig(
                     downstream_package_name=centos_package
                 )
-            else:
+            elif fedora_package:
                 self.fedora_package = (
                     self.fedora_package or local_project.working_dir.name
                 )
                 self.package_config = PackageConfig(
                     downstream_package_name=fedora_package
                 )
+            else:
+                raise PackitException(
+                    "Please tell us the name of the package in the downstream."
+                )
             self.dist_git_path = self.tmpdir.joinpath(
                 self.package_config.downstream_package_name
             )
@@ -180,6 +225,10 @@ class SourceGitGenerator:
     def dist_git(self) -> DistGit:
         if not self._dist_git:
             self._dist_git = self._get_dist_git()
+            # we need to parse the spec twice
+            # https://github.com/rebase-helper/rebase-helper/issues/848
+            self._dist_git.download_remote_sources()
+            self._dist_git.specfile.reload()
         return self._dist_git
 
     @property
@@ -222,11 +271,20 @@ class SourceGitGenerator:
         For given package names, clone the dist-git repo in the given directory
         and return the DistGit class
 
-        :return: DistGit instance (CentOSDistGit if centos_package is set)
+        :return: DistGit instance
         """
         if self.centos_package:
-            self.dist_git_branch = self.dist_git_branch or "c8s"
-            return CentOSDistGit.clone(
+            self.dist_git_branch = self.dist_git_branch or "c9s"
+            # let's be sure to cover anything 9 related,
+            # even though "c9" will probably never be a thing
+            if "c9" in self.dist_git_branch:
+                return CentOS9DistGit.clone(
+                    config=self.config,
+                    package_config=self.package_config,
+                    path=self.dist_git_path,
+                    branch=self.dist_git_branch,
+                )
+            return CentOS8DistGit.clone(
                 config=self.config,
                 package_config=self.package_config,
                 path=self.dist_git_path,
@@ -381,7 +439,7 @@ class SourceGitGenerator:
         self._pull_upstream_ref()
         self._put_downstream_sources()
         self._add_packit_config()
-        if self.dist_git.specfile.patches:
+        if self.dist_git.specfile.get_applied_patches():
             self._run_prep()
             self._rebase_patches("master")
             # TODO: patches which are defined but not applied should be copied
diff --git a/packit/utils/repo.py b/packit/utils/repo.py
index 15f0d621..91f7f4a7 100644
--- a/packit/utils/repo.py
+++ b/packit/utils/repo.py
@@ -10,6 +10,8 @@ from typing import Tuple, Optional, Union, List
 import git
 import subprocess
 from ogr.parsing import parse_git_repo
+
+from packit.constants import CENTOS_DOMAIN, CENTOS_STREAM_GITLAB
 from packit.utils.commands import run_command
 
 from packit.exceptions import PackitException
@@ -105,7 +107,7 @@ def get_current_version_command(
     ]
 
 
-def clone_centos_package(
+def clone_centos_8_package(
     package_name: str,
     dist_git_path: Path,
     branch: str = "c8s",
@@ -121,7 +123,31 @@ def clone_centos_package(
             "clone",
             "-b",
             branch,
-            f"https://git{'.stg' if stg else ''}.centos.org/{namespace}/{package_name}.git",
+            f"https://git{'.stg' if stg else ''}.{CENTOS_DOMAIN}/{namespace}/{package_name}.git",
+            str(dist_git_path),
+        ]
+    )
+
+
+def clone_centos_9_package(
+    package_name: str,
+    dist_git_path: Path,
+    branch: str = "c9s",
+    namespace: str = "rpms",
+    stg: bool = None,
+):
+    """
+    clone selected package from git.[stg.]centos.org
+    """
+    if stg:
+        logger.warning("There is no staging instance for CentOS Stream 9 dist-git.")
+    run_command(
+        [
+            "git",
+            "clone",
+            "-b",
+            branch,
+            f"https://{CENTOS_STREAM_GITLAB}/{namespace}/{package_name}.git",
             str(dist_git_path),
         ]
     )
diff --git a/tests/integration/test_source_git_generator.py b/tests/integration/test_source_git_generator.py
index 2d7e9d24..c3e1f8e0 100644
--- a/tests/integration/test_source_git_generator.py
+++ b/tests/integration/test_source_git_generator.py
@@ -5,6 +5,7 @@ from pathlib import Path
 
 import pytest
 
+from packit.constants import CENTOS_DOMAIN, CENTOS_STREAM_GITLAB
 from packit.fedpkg import FedPKG
 from packit.local_project import LocalProject
 from packit.patches import PatchMetadata
@@ -199,7 +200,12 @@ def test_create_srcgit_requre_populated(api_instance_source_git, tmp_path: Path)
 
 
 @pytest.mark.slow
-def test_centos_cronie(api_instance_source_git, tmp_path: Path):
+@pytest.mark.parametrize(
+    "dist_git_branch,upstream_ref", (("c8s", "cronie-1.5.2"), ("c9s", "cronie-1.5.5"))
+)
+def test_centos_cronie(
+    dist_git_branch, upstream_ref, api_instance_source_git, tmp_path: Path
+):
     source_git_path = tmp_path.joinpath("cronie-sg")
     # create src-git
     source_git_path.mkdir()
@@ -208,12 +214,18 @@ def test_centos_cronie(api_instance_source_git, tmp_path: Path):
         LocalProject(working_dir=source_git_path),
         api_instance_source_git.config,
         "https://github.com/cronie-crond/cronie",
-        upstream_ref="cronie-1.5.2",
+        upstream_ref=upstream_ref,
         centos_package="cronie",
+        dist_git_branch=dist_git_branch,
     )
     sgg.create_from_upstream()
 
+    if dist_git_branch == "c8s":
+        assert CENTOS_DOMAIN in sgg.dist_git.local_project.git_url
+    else:
+        assert CENTOS_STREAM_GITLAB in sgg.dist_git.local_project.git_url
+
     # verify it
     subprocess.check_call(["packit", "srpm"], cwd=source_git_path)
-    srpm_path = list(source_git_path.glob("cronie-1.5.2-*.src.rpm"))[0]
+    srpm_path = list(source_git_path.glob("cronie-*.src.rpm"))[0]
     assert srpm_path.is_file()
diff --git a/tests/unit/test_source_git_generator.py b/tests/unit/test_source_git_generator.py
index 51e2b736..1bc76c9a 100644
--- a/tests/unit/test_source_git_generator.py
+++ b/tests/unit/test_source_git_generator.py
@@ -7,7 +7,7 @@ from flexmock import flexmock
 from packit.config import Config
 from packit.distgit import DistGit
 from packit.local_project import LocalProject
-from packit.source_git import SourceGitGenerator, CentOSDistGit
+from packit.source_git import SourceGitGenerator, CentOS8DistGit
 from packit.utils.repo import create_new_repo
 
 
@@ -26,7 +26,7 @@ def test_centos_conf(cronie, tmp_path: Path):
     )
 
     dg = sgg.dist_git
-    assert isinstance(dg, CentOSDistGit)
+    assert isinstance(dg, CentOS8DistGit)
 
     flexmock(
         DistGit,