Repository: buildbot/buildbot

Issue:
Git: fetch all branches before switching to the given revision, if branch=None
This ticket is a migrated Trac ticket [2750](http://trac.buildbot.net/ticket/2750)

People contributed to the original ticket: @djmitche, @sa2ajj
Ticket created on: `Apr 07 2014`
Ticket last modified on: `Sep 09 2014`

---

See https://github.com/buildbot/buildbot/pull/1074

`branch=None` is a bit of a degenerate case, referring vacuously to the "default branch", which in Git really means master.  However, it's also the default, and there's no reason to make that default fail in the interest of an optimization (not fetching every commit).


---

_Comment from_: @sa2ajj
_Date_: `Sep 08 2014`

Maybe the meaning of `branch=None` should be reconsidered:
* if revision is not specified -> **default branch**
* if a revision is specified -> fetch that particular revision and set it as the one to work on

---
_Comment from_: @dustin
_Date_: `Sep 09 2014`

Maybe I misunderstand, but you can't instruct git to fetch a specific revision from a remote repository -- only to fetch some named revision (and its parents).

---
_Comment from_: @sa2ajj
_Date_: `Sep 09 2014`

Maybe, I misremembered something.

I'll investigate and update the ticket.

---
_Comment from_: @sa2ajj
_Date_: `Sep 09 2014`

It looks like


    git fetch <remote> <SHA-1>:FETCH_HEAD


works just fine.


    $ git --version
    git version 2.1.0


---
_Comment from_: @dustin
_Date_: `Sep 09 2014`

Hah, well, I stand corrected!




Discussion:
Just my 50 cents. Default  branch isn't  always ` master` one :)
Yeah, I think if you're using a branch other than master, then `branch=None` is not a good choice :)

Resolving patch:
diff --git a/master/buildbot/changes/gitpoller.py b/master/buildbot/changes/gitpoller.py
--- a/master/buildbot/changes/gitpoller.py
+++ b/master/buildbot/changes/gitpoller.py
@@ -181,7 +181,7 @@ def reconfigService(
             if only_tags:
                 branches = lambda ref: ref.startswith('refs/tags/')  # noqa: E731
             else:
-                branches = ['master']
+                branches = None
 
         self.repourl = repourl
         self.branches = branches
@@ -256,6 +256,37 @@ def describe(self):
 
         return str
 
+    @async_to_deferred
+    async def _resolve_head_ref(self) -> str | None:
+        if self.supports_lsremote_symref:
+            rows: str = await self._dovccmd('ls-remote', ['--symref', self.repourl, 'HEAD'])
+            # simple parse of output which should have format:
+            # ref: refs/heads/{branch}	HEAD
+            # {hash}	HEAD
+            parts = rows.split(maxsplit=3)
+            # sanity just in case
+            if len(parts) >= 3 and parts[0] == 'ref:' and parts[2] == 'HEAD':
+                return parts[1]
+            return None
+
+        # naive fallback if git version does not support --symref
+        rows = await self._dovccmd('ls-remote', [self.repourl, 'HEAD', 'refs/heads/*'])
+        refs = [row.split('\t') for row in rows.splitlines() if '\t' in row]
+        # retrieve hash that HEAD points to
+        head_hash = next((hash for hash, ref in refs if ref == 'HEAD'), None)
+        if head_hash is None:
+            return None
+
+        # get refs that points to the same hash as HEAD
+        candidates = [ref for hash, ref in refs if ref != 'HEAD' and hash == head_hash]
+        # Found default branch
+        if len(candidates) == 1:
+            return candidates[0]
+
+        # If multiple ref points to the same hash as HEAD,
+        # we have no way to know which one is the default
+        return None
+
     @async_to_deferred
     async def _get_refs(self, refs: list[str] | None = None) -> list[str]:
         rows: str = await self._dovccmd(
@@ -337,6 +368,14 @@ def poll(self):
         elif self.branches:
             refs = yield self._get_refs([f"refs/heads/{b}" for b in self.branches])
             trim_ref_head = True
+        else:
+            head_ref = yield self._resolve_head_ref()
+            if head_ref is not None:
+                refs = [head_ref]
+            else:
+                # unlikely, but if we can't find HEAD here, something weird happen,
+                # but not a critical error. Just use HEAD as the ref to use
+                refs = ['HEAD']
 
         # Nothing to fetch and process.
         if not refs:
diff --git a/master/buildbot/util/git.py b/master/buildbot/util/git.py
--- a/master/buildbot/util/git.py
+++ b/master/buildbot/util/git.py
@@ -88,6 +88,7 @@ def setupGit(self):
         self.supportsSshPrivateKeyAsEnvOption = False
         self.supportsSshPrivateKeyAsConfigOption = False
         self.supportsFilters = False
+        self.supports_lsremote_symref = False
 
     def parseGitFeatures(self, version_stdout):
         match = re.match(r"^git version (\d+(\.\d+)*)", version_stdout)
@@ -107,6 +108,9 @@ def parseGitFeatures(self, version_stdout):
             self.supportsSubmoduleCheckout = True
         if version >= parse_version("2.3.0"):
             self.supportsSshPrivateKeyAsEnvOption = True
+        if version >= parse_version("2.8.0"):
+            # https://github.com/git/git/blob/v2.8.0/Documentation/RelNotes/2.8.0.txt#L72-L73
+            self.supports_lsremote_symref = True
         if version >= parse_version("2.10.0"):
             self.supportsSshPrivateKeyAsConfigOption = True
         if version >= parse_version("2.27.0"):


Test patch:
diff --git a/master/buildbot/test/unit/changes/test_gitpoller.py b/master/buildbot/test/unit/changes/test_gitpoller.py
--- a/master/buildbot/test/unit/changes/test_gitpoller.py
+++ b/master/buildbot/test/unit/changes/test_gitpoller.py
@@ -54,7 +54,7 @@ class TestGitPollerBase(
 
     def createPoller(self):
         # this is overridden in TestGitPollerWithSshPrivateKey
-        return gitpoller.GitPoller(self.REPOURL)
+        return gitpoller.GitPoller(self.REPOURL, branches=['master'])
 
     @defer.inlineCallbacks
     def setUp(self):
@@ -1863,9 +1863,175 @@ def test_startService_loadLastRev(self):
         yield self.assert_last_rev({"master": "fa3ae8ed68e664d4db24798611b352e3c6509930"})
 
 
+class TestGitPollerDefaultBranch(TestGitPollerBase):
+    def createPoller(self):
+        return gitpoller.GitPoller(self.REPOURL, branches=None)
+
+    @async_to_deferred
+    async def test_resolve_head_ref_with_symref(self):
+        self.patch(self.poller, 'supports_lsremote_symref', True)
+
+        self.expect_commands(
+            ExpectMasterShell(['git', 'ls-remote', '--symref', self.REPOURL, 'HEAD'])
+            .exit(0)
+            .stdout(
+                b'ref: refs/heads/default_branch	HEAD\n'
+                b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09	HEAD\n'
+            ),
+        )
+
+        result = await self.poller._resolve_head_ref()
+
+        self.assert_all_commands_ran()
+        self.assertEqual(result, 'refs/heads/default_branch')
+
+    @async_to_deferred
+    async def test_resolve_head_ref_without_symref(self):
+        self.patch(self.poller, 'supports_lsremote_symref', False)
+
+        self.expect_commands(
+            ExpectMasterShell(['git', 'ls-remote', self.REPOURL, 'HEAD', 'refs/heads/*'])
+            .exit(0)
+            .stdout(
+                b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09	HEAD\n'
+                b'274ec17f8bfb56adc0035b12735785097df488fc	refs/heads/3.10.x\n'
+                b'972a389242fd15a59f2d2840d1be4c0fc7b97109	refs/heads/3.11.x\n'
+                b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09	refs/heads/master\n'
+            ),
+        )
+
+        result = await self.poller._resolve_head_ref()
+
+        self.assert_all_commands_ran()
+        self.assertEqual(result, 'refs/heads/master')
+
+    @async_to_deferred
+    async def test_resolve_head_ref_without_symref_multiple_head_candidates(self):
+        self.patch(self.poller, 'supports_lsremote_symref', False)
+
+        self.expect_commands(
+            ExpectMasterShell(['git', 'ls-remote', self.REPOURL, 'HEAD', 'refs/heads/*'])
+            .exit(0)
+            .stdout(
+                b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09	HEAD\n'
+                b'274ec17f8bfb56adc0035b12735785097df488fc	refs/heads/3.10.x\n'
+                b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09	refs/heads/3.11.x\n'
+                b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09	refs/heads/master\n'
+            ),
+        )
+
+        result = await self.poller._resolve_head_ref()
+
+        self.assert_all_commands_ran()
+        self.assertEqual(result, None)
+
+    @async_to_deferred
+    async def test_poll_found_head(self):
+        self.expect_commands(
+            ExpectMasterShell(['git', '--version']).stdout(b'git version 2.10.0\n'),
+            ExpectMasterShell(['git', 'init', '--bare', self.POLLER_WORKDIR]),
+            ExpectMasterShell([
+                'git',
+                'ls-remote',
+                '--symref',
+                self.REPOURL,
+                'HEAD',
+            ]).stdout(
+                b'ref: refs/heads/default_branch	HEAD\n'
+                b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09	HEAD\n'
+            ),
+            ExpectMasterShell([
+                'git',
+                'fetch',
+                '--progress',
+                self.REPOURL,
+                f'+refs/heads/default_branch:refs/buildbot/{self.REPOURL_QUOTED}/heads/default_branch',
+                '--',
+            ]).workdir(self.POLLER_WORKDIR),
+            ExpectMasterShell([
+                'git',
+                'rev-parse',
+                f'refs/buildbot/{self.REPOURL_QUOTED}/heads/default_branch',
+                '--',
+            ])
+            .workdir(self.POLLER_WORKDIR)
+            .stdout(b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09\n'),
+            ExpectMasterShell([
+                'git',
+                'log',
+                '--ignore-missing',
+                '--format=%H',
+                '737b94eca1ddde3dd4a0040b25c8a25fe973fe09',
+                '^4423cdbcbb89c14e50dd5f4152415afd686c5241',
+                '--',
+            ]).workdir(self.POLLER_WORKDIR),
+        )
+
+        await self.set_last_rev({
+            'master': '4423cdbcbb89c14e50dd5f4152415afd686c5241',
+        })
+        self.poller.doPoll.running = True
+        await self.poller.poll()
+
+        self.assert_all_commands_ran()
+        await self.assert_last_rev({
+            'refs/heads/default_branch': '737b94eca1ddde3dd4a0040b25c8a25fe973fe09'
+        })
+        self.assertEqual(len(self.master.data.updates.changesAdded), 0)
+
+    @async_to_deferred
+    async def test_poll_found_head_not_found(self):
+        self.expect_commands(
+            ExpectMasterShell(['git', '--version']).stdout(b'git version 2.10.0\n'),
+            ExpectMasterShell(['git', 'init', '--bare', self.POLLER_WORKDIR]),
+            ExpectMasterShell([
+                'git',
+                'ls-remote',
+                '--symref',
+                self.REPOURL,
+                'HEAD',
+            ]).stdout(b'malformed output'),
+            ExpectMasterShell([
+                'git',
+                'fetch',
+                '--progress',
+                self.REPOURL,
+                f'+HEAD:refs/buildbot/raw/{self.REPOURL_QUOTED}/HEAD',
+                '--',
+            ]).workdir(self.POLLER_WORKDIR),
+            ExpectMasterShell([
+                'git',
+                'rev-parse',
+                f'refs/buildbot/raw/{self.REPOURL_QUOTED}/HEAD',
+                '--',
+            ])
+            .workdir(self.POLLER_WORKDIR)
+            .stdout(b'737b94eca1ddde3dd4a0040b25c8a25fe973fe09\n'),
+            ExpectMasterShell([
+                'git',
+                'log',
+                '--ignore-missing',
+                '--format=%H',
+                '737b94eca1ddde3dd4a0040b25c8a25fe973fe09',
+                '^4423cdbcbb89c14e50dd5f4152415afd686c5241',
+                '--',
+            ]).workdir(self.POLLER_WORKDIR),
+        )
+
+        await self.set_last_rev({
+            'master': '4423cdbcbb89c14e50dd5f4152415afd686c5241',
+        })
+        self.poller.doPoll.running = True
+        await self.poller.poll()
+
+        self.assert_all_commands_ran()
+        await self.assert_last_rev({'HEAD': '737b94eca1ddde3dd4a0040b25c8a25fe973fe09'})
+        self.assertEqual(len(self.master.data.updates.changesAdded), 0)
+
+
 class TestGitPollerWithSshPrivateKey(TestGitPollerBase):
     def createPoller(self):
-        return gitpoller.GitPoller(self.REPOURL, sshPrivateKey='ssh-key')
+        return gitpoller.GitPoller(self.REPOURL, branches=['master'], sshPrivateKey='ssh-key')
 
     @mock.patch(
         'buildbot.util.private_tempdir.PrivateTemporaryDirectory',
@@ -2033,7 +2199,9 @@ def test_poll_failFetch_git_2_10(self, write_local_file_mock, temp_dir_mock):
 
 class TestGitPollerWithSshHostKey(TestGitPollerBase):
     def createPoller(self):
-        return gitpoller.GitPoller(self.REPOURL, sshPrivateKey='ssh-key', sshHostKey='ssh-host-key')
+        return gitpoller.GitPoller(
+            self.REPOURL, branches=['master'], sshPrivateKey='ssh-key', sshHostKey='ssh-host-key'
+        )
 
     @mock.patch(
         'buildbot.util.private_tempdir.PrivateTemporaryDirectory',
@@ -2103,7 +2271,10 @@ def test_poll_initial_2_10(self, write_local_file_mock, temp_dir_mock):
 class TestGitPollerWithSshKnownHosts(TestGitPollerBase):
     def createPoller(self):
         return gitpoller.GitPoller(
-            self.REPOURL, sshPrivateKey='ssh-key\n', sshKnownHosts='ssh-known-hosts'
+            self.REPOURL,
+            branches=['master'],
+            sshPrivateKey='ssh-key\n',
+            sshKnownHosts='ssh-known-hosts',
         )
 
     @mock.patch(
@@ -2207,7 +2378,7 @@ def test_deprecated_pollinterval(self):
     @defer.inlineCallbacks
     def test_branches_default(self):
         poller = yield self.attachChangeSource(gitpoller.GitPoller("/tmp/git.git"))
-        self.assertEqual(poller.branches, ["master"])
+        self.assertEqual(poller.branches, None)
 
     @defer.inlineCallbacks
     def test_branches_oldBranch(self):
@@ -2312,3 +2483,9 @@ def test_tracker_ref_not_ref_collision(self):
                 "https://example.org/repo.git", "refs/heads/branch_name"
             ),
         )
+
+    def test_tracker_ref_HEAD(self):
+        self.assertNotEqual(
+            gitpoller.GitPoller._tracker_ref("https://example.org/repo.git", "HEAD"),
+            gitpoller.GitPoller._tracker_ref("https://example.org/repo.git", "refs/raw/HEAD"),
+        )
diff --git a/master/buildbot/test/unit/util/test_git.py b/master/buildbot/test/unit/util/test_git.py
--- a/master/buildbot/test/unit/util/test_git.py
+++ b/master/buildbot/test/unit/util/test_git.py
@@ -112,6 +112,7 @@ def test_no_output(self):
         self.assertFalse(self.supportsSubmoduleCheckout)
         self.assertFalse(self.supportsSshPrivateKeyAsEnvOption)
         self.assertFalse(self.supportsSshPrivateKeyAsConfigOption)
+        self.assertFalse(self.supports_lsremote_symref)
 
     def test_git_noversion(self):
         self.parseGitFeatures('git')
@@ -121,6 +122,7 @@ def test_git_noversion(self):
         self.assertFalse(self.supportsSubmoduleCheckout)
         self.assertFalse(self.supportsSshPrivateKeyAsEnvOption)
         self.assertFalse(self.supportsSshPrivateKeyAsConfigOption)
+        self.assertFalse(self.supports_lsremote_symref)
 
     def test_git_zero_version(self):
         self.parseGitFeatures('git version 0.0.0')
@@ -130,6 +132,7 @@ def test_git_zero_version(self):
         self.assertFalse(self.supportsSubmoduleCheckout)
         self.assertFalse(self.supportsSshPrivateKeyAsEnvOption)
         self.assertFalse(self.supportsSshPrivateKeyAsConfigOption)
+        self.assertFalse(self.supports_lsremote_symref)
 
     def test_git_2_10_0(self):
         self.parseGitFeatures('git version 2.10.0')
@@ -139,6 +142,7 @@ def test_git_2_10_0(self):
         self.assertTrue(self.supportsSubmoduleCheckout)
         self.assertTrue(self.supportsSshPrivateKeyAsEnvOption)
         self.assertTrue(self.supportsSshPrivateKeyAsConfigOption)
+        self.assertTrue(self.supports_lsremote_symref)
 
 
 class TestAdjustCommandParamsForSshPrivateKey(GitMixin, unittest.TestCase):