Repository: buildbot/buildbot

Issue:
GitPoller fails with filename too long when using Bitbucket repository tokens
We've hit an issue with GitPoller as we try to move from away from app passwords for our CI jobs to using repository tokens.  The issue seems to be that GitPoller uses `git fetch <repo_url> +<src>:<dst>` where the `<dst>` directory uses an escaped version of the full repo URL.  When using repository tokens, the URL is of the form `https://x-token-auth:ReallyLongToken@bitbucket.org/org/repo.git` and the full destination refspec becomes too long.

Is there any reason that the full URL is used in the destination refspec here:  https://github.com/buildbot/buildbot/blob/master/master/buildbot/changes/gitpoller.py#L209

Is it just creating a usefully named destination directory and if so, could we detect the use of a token and just strip it?  Or are there nicer ways of solving this?


Discussion:

Resolving patch:
diff --git a/master/buildbot/changes/gitpoller.py b/master/buildbot/changes/gitpoller.py
--- a/master/buildbot/changes/gitpoller.py
+++ b/master/buildbot/changes/gitpoller.py
@@ -26,6 +26,7 @@
 from buildbot import config
 from buildbot.changes import base
 from buildbot.util import bytes2unicode
+from buildbot.util import giturlparse
 from buildbot.util import private_tempdir
 from buildbot.util import runprocess
 from buildbot.util.git import GitMixin
@@ -281,9 +282,30 @@ def _removeHeads(self, branch):
             branch = branch[11:]
         return branch
 
-    def _trackerBranch(self, branch):
-        url = urlquote(self.repourl, '').replace('~', '%7E')
-        return f"refs/buildbot/{url}/{self._trim_prefix(branch, 'refs/')}"
+    @staticmethod
+    def _tracker_ref(repourl: str, ref: str) -> str:
+        def _sanitize(value: str) -> str:
+            return urlquote(value, '').replace('~', '%7E')
+
+        tracker_prefix = "refs/buildbot"
+        # if ref is not a Git ref, store under a different path to avoid collision
+        if not ref.startswith('refs/'):
+            tracker_prefix += "/raw"
+
+        git_url = giturlparse(repourl)
+        if git_url is None:
+            # fallback to using the whole repourl
+            url_identifier = _sanitize(repourl)
+        else:
+            url_identifier = f"{git_url.proto}/{_sanitize(git_url.domain)}"
+            if git_url.port is not None:
+                url_identifier += f":{git_url.port}"
+
+            if git_url.owner is not None:
+                url_identifier += f"/{_sanitize(git_url.owner)}"
+            url_identifier += f"/{_sanitize(git_url.repo)}"
+
+        return f"{tracker_prefix}/{url_identifier}/{GitPoller._trim_prefix(ref, 'refs/')}"
 
     def poll_should_exit(self):
         # A single gitpoller loop may take a while on a loaded master, which would block
@@ -320,7 +342,7 @@ def poll(self):
         if self.poll_should_exit():
             return
 
-        refspecs = [f'+{ref}:{self._trackerBranch(ref)}' for ref in refs]
+        refspecs = [f'+{ref}:{self._tracker_ref(self.repourl, ref)}' for ref in refs]
 
         try:
             yield self._dovccmd(
@@ -344,7 +366,7 @@ def poll(self):
                     break
 
                 rev = yield self._dovccmd(
-                    'rev-parse', [self._trackerBranch(ref), '--'], path=self.workdir
+                    'rev-parse', [self._tracker_ref(self.repourl, ref), '--'], path=self.workdir
                 )
                 revs[branch] = rev
                 yield self._process_changes(rev, branch)
diff --git a/master/buildbot/util/giturlparse.py b/master/buildbot/util/giturlparse.py
--- a/master/buildbot/util/giturlparse.py
+++ b/master/buildbot/util/giturlparse.py
@@ -13,9 +13,10 @@
 #
 # Copyright Buildbot Team Members
 
+from __future__ import annotations
 
 import re
-from collections import namedtuple
+from typing import NamedTuple
 
 # The regex is matching more than it should and is not intended to be an url validator.
 # It is intended to efficiently and reliably extract information from the various examples
@@ -23,15 +24,23 @@
 
 _giturlmatcher = re.compile(
     r'(?P<proto>(https?://|ssh://|git://|))'
-    r'((?P<user>.*)@)?'
+    r'((?P<user>[^:@]*)(:(?P<password>.*))?@)?'
     r'(?P<domain>[^\/:]+)(:((?P<port>[0-9]+)/)?|/)'
     r'((?P<owner>.+)/)?(?P<repo>[^/]+?)(\.git)?$'
 )
 
-GitUrl = namedtuple('GitUrl', ['proto', 'user', 'domain', 'port', 'owner', 'repo'])
 
+class GitUrl(NamedTuple):
+    proto: str
+    user: str | None
+    password: str | None
+    domain: str
+    port: int | None
+    owner: str | None
+    repo: str
 
-def giturlparse(url):
+
+def giturlparse(url: str) -> GitUrl | None:
     res = _giturlmatcher.match(url)
     if res is None:
         return None
@@ -45,5 +54,11 @@ def giturlparse(url):
     else:
         proto = 'ssh'  # implicit proto is ssh
     return GitUrl(
-        proto, res.group('user'), res.group("domain"), port, res.group('owner'), res.group('repo')
+        proto=proto,
+        user=res.group('user'),
+        password=res.group('password'),
+        domain=res.group("domain"),
+        port=port,
+        owner=res.group('owner'),
+        repo=res.group('repo'),
     )


Test patch:
diff --git a/master/buildbot/test/unit/changes/test_gitpoller.py b/master/buildbot/test/unit/changes/test_gitpoller.py
--- a/master/buildbot/test/unit/changes/test_gitpoller.py
+++ b/master/buildbot/test/unit/changes/test_gitpoller.py
@@ -46,7 +46,7 @@ class TestGitPollerBase(
     unittest.TestCase,
 ):
     REPOURL = 'git@example.com:~foo/baz.git'
-    REPOURL_QUOTED = 'git%40example.com%3A%7Efoo%2Fbaz.git'
+    REPOURL_QUOTED = 'ssh/example.com/%7Efoo/baz'
 
     POLLER_WORKDIR = os.path.join('basedir', 'gitpoller-work')
 
@@ -2247,3 +2247,59 @@ def test_branch_and_only_tags(self):
     def test_gitbin_default(self):
         poller = yield self.attachChangeSource(gitpoller.GitPoller("/tmp/git.git"))
         self.assertEqual(poller.gitbin, "git")
+
+
+class TestGitPollerUtils(unittest.TestCase):
+    def test_tracker_ref_protos(self):
+        for url, expected_tracker in [
+            (
+                "https://example.org/owner/repo.git",
+                "refs/buildbot/https/example.org/owner/repo/heads/branch_name",
+            ),
+            ("ssh://example.org:repo.git", "refs/buildbot/ssh/example.org/repo/heads/branch_name"),
+            ("git@example.org:repo.git", "refs/buildbot/ssh/example.org/repo/heads/branch_name"),
+        ]:
+            self.assertEqual(
+                gitpoller.GitPoller._tracker_ref(url, "refs/heads/branch_name"),
+                expected_tracker,
+            )
+
+    def test_tracker_ref_with_port(self):
+        self.assertEqual(
+            gitpoller.GitPoller._tracker_ref(
+                "https://example.org:1234/owner/repo.git", "refs/heads/branch_name"
+            ),
+            "refs/buildbot/https/example.org:1234/owner/repo/heads/branch_name",
+        )
+
+    def test_tracker_ref_tag(self):
+        self.assertEqual(
+            gitpoller.GitPoller._tracker_ref(
+                "https://example.org:1234/owner/repo.git", "refs/tags/v1"
+            ),
+            "refs/buildbot/https/example.org:1234/owner/repo/tags/v1",
+        )
+
+    def test_tracker_ref_with_credentials(self):
+        self.assertEqual(
+            gitpoller.GitPoller._tracker_ref(
+                "https://user:password@example.org:1234/owner/repo.git", "refs/heads/branch_name"
+            ),
+            "refs/buildbot/https/example.org:1234/owner/repo/heads/branch_name",
+        )
+
+    def test_tracker_ref_sub_branch(self):
+        self.assertEqual(
+            gitpoller.GitPoller._tracker_ref(
+                "https://user:password@example.org:1234/owner/repo.git", "refs/heads/branch_name"
+            ),
+            "refs/buildbot/https/example.org:1234/owner/repo/heads/branch_name",
+        )
+
+    def test_tracker_ref_not_ref_collision(self):
+        self.assertNotEqual(
+            gitpoller.GitPoller._tracker_ref("https://example.org/repo.git", "heads/branch_name"),
+            gitpoller.GitPoller._tracker_ref(
+                "https://example.org/repo.git", "refs/heads/branch_name"
+            ),
+        )
diff --git a/master/buildbot/test/unit/util/test_giturlparse.py b/master/buildbot/test/unit/util/test_giturlparse.py
--- a/master/buildbot/test/unit/util/test_giturlparse.py
+++ b/master/buildbot/test/unit/util/test_giturlparse.py
@@ -111,3 +111,16 @@ def test_protos(self):
         self.assertEqual(giturlparse("git://bitbucket.org/org/repo.git").proto, "git")
         self.assertEqual(giturlparse("ssh://git@bitbucket.org:org/repo.git").proto, "ssh")
         self.assertEqual(giturlparse("git@bitbucket.org:org/repo.git").proto, "ssh")
+
+    def test_user_password(self):
+        for u, expected_user, expected_password in [
+            ("https://user@github.com/buildbot/buildbot", "user", None),
+            ("https://user:password@github.com/buildbot/buildbot", "user", "password"),
+        ]:
+            u = giturlparse(u)
+            self.assertEqual(u.user, expected_user)
+            self.assertEqual(u.password, expected_password)
+            self.assertEqual(u.domain, "github.com")
+            self.assertEqual(u.owner, "buildbot")
+            self.assertEqual(u.repo, "buildbot")
+            self.assertIsNone(u.port)