Repository: mesonbuild/meson

Issue:
dist failure with relative submodule paths
**Describe the bug**

dist fails with git submodules using relative paths.

**To Reproduce**

The entire tree is at https://gitlab.freedesktop.org/xdg/shared-mime-info/-/tree/40db4c7adba8a5cc586e24f5b4b5aff0b2a81bcc, see .gitlab-ci.yml there for how it's built in CI. The xdgmime submodule is defined as `../xdgmime.git`

CI failure log is at https://gitlab.freedesktop.org/xdg/shared-mime-info/-/jobs/6403481 -- essentially it seems meson is trying to checkout the submodule from a directory relative to the main project working dir, whereas it should treat relative submodule path as relative to the superproject's default remote repo; https://git-scm.com/docs/git-submodule#Documentation/git-submodule.txt-add-bltbranchgt-f--force--nameltnamegt--referenceltrepositorygt--depthltdepthgt--ltrepositorygtltpathgt

**Expected behavior**

Working submodule checkout during dist.

**system parameters**
* Is this a [cross build](https://mesonbuild.com/Cross-compilation.html) or just a plain native build (for the same computer)?
* what operating system (e.g. MacOS Catalina, Windows 10, CentOS 8.0, Ubuntu 18.04, etc.)
* what Python version are you using e.g. 3.8.0
* what `meson --version`
* what `ninja --version` if it's a Ninja build



Discussion:

Resolving patch:
diff --git a/mesonbuild/mdist.py b/mesonbuild/mdist.py
--- a/mesonbuild/mdist.py
+++ b/mesonbuild/mdist.py
@@ -18,6 +18,8 @@
 import sys
 import shutil
 import subprocess
+import tarfile
+import tempfile
 import hashlib
 import json
 from glob import glob
@@ -56,30 +58,36 @@ def create_hash(fname):
         f.write('{} *{}\n'.format(m.hexdigest(), os.path.basename(fname)))
 
 
-def del_gitfiles(dirname):
-    gitfiles = ('.git', '.gitattributes', '.gitignore', '.gitmodules')
-    for f in glob(os.path.join(dirname, '.git*')):
-        if os.path.split(f)[1] in gitfiles:
-            if os.path.isdir(f) and not os.path.islink(f):
-                windows_proof_rmtree(f)
-            else:
-                os.unlink(f)
-
-def process_submodules(dirname):
-    module_file = os.path.join(dirname, '.gitmodules')
+def copy_git(src, distdir, revision='HEAD', prefix=None, subdir=None):
+    cmd = ['git', 'archive', '--format', 'tar', revision]
+    if prefix is not None:
+        cmd.insert(2, f'--prefix={prefix}/')
+    if subdir is not None:
+        cmd.extend(['--', subdir])
+    with tempfile.TemporaryFile() as f:
+        subprocess.check_call(cmd, cwd=src, stdout=f)
+        f.seek(0)
+        t = tarfile.open(fileobj=f) # [ignore encoding]
+        t.extractall(path=distdir)
+
+def process_submodules(src, distdir):
+    module_file = os.path.join(src, '.gitmodules')
     if not os.path.exists(module_file):
         return
-    subprocess.check_call(['git', 'submodule', 'update', '--init', '--recursive'], cwd=dirname)
-    for line in open(module_file, encoding='utf-8'):
-        line = line.strip()
-        if '=' not in line:
-            continue
-        k, v = line.split('=', 1)
-        k = k.strip()
-        v = v.strip()
-        if k != 'path':
+    cmd = ['git', 'submodule', 'status', '--cached', '--recursive']
+    modlist = subprocess.check_output(cmd, cwd=src, universal_newlines=True).splitlines()
+    for submodule in modlist:
+        status = submodule[:1]
+        sha1, rest = submodule[1:].split(' ', 1)
+        subpath = rest.rsplit(' ', 1)[0]
+
+        if status == '-':
+            mlog.warning(f'Submodule {subpath!r} is not checked out and cannot be added to the dist')
             continue
-        del_gitfiles(os.path.join(dirname, v))
+        elif status in {'+', 'U'}:
+            mlog.warning(f'Submodule {subpath!r} has uncommitted changes that will not be included in the dist tarball')
+
+        copy_git(os.path.join(src, subpath), distdir, revision=sha1, prefix=subpath)
 
 
 def run_dist_scripts(src_root, bld_root, dist_root, dist_scripts, subprojects):
@@ -126,7 +134,7 @@ def git_have_dirty_index(src_root):
     ret = subprocess.call(['git', '-C', src_root, 'diff-index', '--quiet', 'HEAD'])
     return ret == 1
 
-def git_clone(src_root, distdir):
+def process_git_project(src_root, distdir):
     if git_have_dirty_index(src_root):
         mlog.warning('Repository has uncommitted changes that will not be included in the dist tarball')
     if os.path.exists(distdir):
@@ -134,30 +142,28 @@ def git_clone(src_root, distdir):
     repo_root = git_root(src_root)
     if repo_root.samefile(src_root):
         os.makedirs(distdir)
-        subprocess.check_call(['git', 'clone', '--shared', src_root, distdir])
+        copy_git(src_root, distdir)
     else:
         subdir = Path(src_root).relative_to(repo_root)
         tmp_distdir = distdir + '-tmp'
         if os.path.exists(tmp_distdir):
             windows_proof_rmtree(tmp_distdir)
         os.makedirs(tmp_distdir)
-        subprocess.check_call(['git', 'clone', '--shared', '--no-checkout', str(repo_root), tmp_distdir])
-        subprocess.check_call(['git', 'checkout', 'HEAD', '--', str(subdir)], cwd=tmp_distdir)
+        copy_git(repo_root, tmp_distdir, subdir=str(subdir))
         Path(tmp_distdir, subdir).rename(distdir)
         windows_proof_rmtree(tmp_distdir)
-    process_submodules(distdir)
-    del_gitfiles(distdir)
+    process_submodules(src_root, distdir)
 
 def create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, dist_scripts, subprojects):
     distdir = os.path.join(dist_sub, dist_name)
-    git_clone(src_root, distdir)
+    process_git_project(src_root, distdir)
     for path in subprojects.values():
         sub_src_root = os.path.join(src_root, path)
         sub_distdir = os.path.join(distdir, path)
         if os.path.exists(sub_distdir):
             continue
         if is_git(sub_src_root):
-            git_clone(sub_src_root, sub_distdir)
+            process_git_project(sub_src_root, sub_distdir)
         else:
             shutil.copytree(sub_src_root, sub_distdir)
     run_dist_scripts(src_root, bld_root, distdir, dist_scripts, subprojects)
diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py
--- a/unittests/allplatformstests.py
+++ b/unittests/allplatformstests.py
@@ -101,7 +101,7 @@ def _git_init(project_dir):
 def _git_add_all(project_dir):
     subprocess.check_call('git add *', cwd=project_dir, shell=True,
                           stdout=subprocess.DEVNULL)
-    subprocess.check_call(['git', 'commit', '-a', '-m', 'I am a project'], cwd=project_dir,
+    subprocess.check_call(['git', 'commit', '--no-gpg-sign', '-a', '-m', 'I am a project'], cwd=project_dir,
                           stdout=subprocess.DEVNULL)
 
 class AllPlatformTests(BasePlatformTests):
diff --git a/unittests/subprojectscommandtests.py b/unittests/subprojectscommandtests.py
--- a/unittests/subprojectscommandtests.py
+++ b/unittests/subprojectscommandtests.py
@@ -83,7 +83,7 @@ def _git_create_repo(self, path):
         self._git(['init'] + extra_cmd, path)
         self._git_config(path)
         self._git(['add', '.'], path)
-        self._git(['commit', '-m', 'Initial commit'], path)
+        self._git(['commit', '--no-gpg-sign', '-m', 'Initial commit'], path)
 
     def _git_create_remote_repo(self, name):
         self._git_create_repo(self.root_dir / name)
@@ -93,15 +93,15 @@ def _git_create_local_repo(self, name):
 
     def _git_create_remote_commit(self, name, branch):
         self._git_remote(['checkout', branch], name)
-        self._git_remote(['commit', '--allow-empty', '-m', f'initial {branch} commit'], name)
+        self._git_remote(['commit', '--no-gpg-sign', '--allow-empty', '-m', f'initial {branch} commit'], name)
 
     def _git_create_remote_branch(self, name, branch):
         self._git_remote(['checkout', '-b', branch], name)
-        self._git_remote(['commit', '--allow-empty', '-m', f'initial {branch} commit'], name)
+        self._git_remote(['commit', '--no-gpg-sign', '--allow-empty', '-m', f'initial {branch} commit'], name)
 
     def _git_create_remote_tag(self, name, tag):
-        self._git_remote(['commit', '--allow-empty', '-m', f'tag {tag} commit'], name)
-        self._git_remote(['tag', tag], name)
+        self._git_remote(['commit', '--no-gpg-sign', '--allow-empty', '-m', f'tag {tag} commit'], name)
+        self._git_remote(['tag', '--no-sign', tag], name)
 
     def _wrap_create_git(self, name, revision='master'):
         path = self.root_dir / name


Test patch: