Repository: easybuilders/easybuild-framework

Issue:
git_clone and the .git directory
When using git_config on sources the tar file created excludes the .git directory in:

.../EasyBuild/4.1.1/lib/python2.7/site-packages/easybuild/tools/filetools.py

on the line:

\# create an archive and delete the git repo directory
tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]

Some software uses the .git files to determine its own version and the directory is crucial for its correct compilation. There is no flag to alter this behaviour. Can this be changed?


Discussion:

Resolving patch:
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -1864,6 +1864,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
     repo_name = git_config.pop('repo_name', None)
     commit = git_config.pop('commit', None)
     recursive = git_config.pop('recursive', False)
+    keep_git_dir = git_config.pop('keep_git_dir', False)
 
     # input validation of git_config dict
     if git_config:
@@ -1912,7 +1913,10 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
         run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name)
 
     # create an archive and delete the git repo directory
-    tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]
+    if keep_git_dir:
+        tar_cmd = ['tar', 'cfvz', targetpath, repo_name]
+    else:
+        tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]
     run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False)
 
     # cleanup (repo_name dir does not exist in dry run mode)


Test patch:
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -268,7 +268,7 @@ def test_checksums(self):
 
         # checksum of length 32 is assumed to be MD5, length 64 to be SHA256, other lengths not allowed
         # checksum of length other than 32/64 yields an error
-        error_pattern = "Length of checksum '.*' \(\d+\) does not match with either MD5 \(32\) or SHA256 \(64\)"
+        error_pattern = r"Length of checksum '.*' \(\d+\) does not match with either MD5 \(32\) or SHA256 \(64\)"
         for checksum in ['tooshort', 'inbetween32and64charactersisnotgoodeither', known_checksums['sha256'] + 'foo']:
             self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum)
 
@@ -584,7 +584,7 @@ def test_read_write_file(self):
 
         txt2 = '\n'.join(['test', '123'])
         ft.write_file(fp, txt2, append=True)
-        self.assertEqual(ft.read_file(fp), txt+txt2)
+        self.assertEqual(ft.read_file(fp), txt + txt2)
 
         # test backing up of existing file
         ft.write_file(fp, 'foo', backup=True)
@@ -1800,7 +1800,7 @@ def test_move_file(self):
         self.mock_stderr(False)
 
         # informative message printed, but file was not actually moved
-        regex = re.compile("^moved file .*/test\.txt to .*/new_test\.txt$")
+        regex = re.compile(r"^moved file .*/test\.txt to .*/new_test\.txt$")
         self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
         self.assertEqual(stderr, '')
 
@@ -1863,7 +1863,7 @@ def test_diff_files(self):
         ])
         res = ft.diff_files(foo, bar)
         self.assertTrue(res.endswith(expected), "%s ends with %s" % (res, expected))
-        regex = re.compile('^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M)
+        regex = re.compile(r'^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M)
         self.assertTrue(regex.search(res), "Pattern '%s' found in: %s" % (regex.pattern, res))
 
     def test_get_source_tarball_from_git(self):
@@ -1954,42 +1954,52 @@ def run_check():
             'tag': 'master',
         }
         expected = '\n'.join([
-            '  running command "git clone --branch master git@github.com:hpcugent/testrepository.git"',
-            "  \(in .*/tmp.*\)",
-            '  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
-            "  \(in .*/tmp.*\)",
+            r'  running command "git clone --branch master git@github.com:hpcugent/testrepository.git"',
+            r"  \(in .*/tmp.*\)",
+            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+            r"  \(in .*/tmp.*\)",
         ])
         run_check()
 
         git_config['recursive'] = True
         expected = '\n'.join([
-            '  running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"',
-            "  \(in .*/tmp.*\)",
-            '  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
-            "  \(in .*/tmp.*\)",
+            r'  running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"',
+            r"  \(in .*/tmp.*\)",
+            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+            r"  \(in .*/tmp.*\)",
         ])
         run_check()
 
+        git_config['keep_git_dir'] = True
+        expected = '\n'.join([
+            r'  running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"',
+            r"  \(in .*/tmp.*\)",
+            r'  running command "tar cfvz .*/target/test.tar.gz testrepository"',
+            r"  \(in .*/tmp.*\)",
+        ])
+        run_check()
+        del git_config['keep_git_dir']
+
         del git_config['tag']
         git_config['commit'] = '8456f86'
         expected = '\n'.join([
-            '  running command "git clone --recursive git@github.com:hpcugent/testrepository.git"',
-            "  \(in .*/tmp.*\)",
-            '  running command "git checkout 8456f86 && git submodule update"',
-            "  \(in testrepository\)",
-            '  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
-            "  \(in .*/tmp.*\)",
+            r'  running command "git clone --recursive git@github.com:hpcugent/testrepository.git"',
+            r"  \(in .*/tmp.*\)",
+            r'  running command "git checkout 8456f86 && git submodule update"',
+            r"  \(in testrepository\)",
+            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+            r"  \(in .*/tmp.*\)",
         ])
         run_check()
 
         del git_config['recursive']
         expected = '\n'.join([
-            '  running command "git clone git@github.com:hpcugent/testrepository.git"',
-            "  \(in .*/tmp.*\)",
-            '  running command "git checkout 8456f86"',
-            "  \(in testrepository\)",
-            '  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
-            "  \(in .*/tmp.*\)",
+            r'  running command "git clone git@github.com:hpcugent/testrepository.git"',
+            r"  \(in .*/tmp.*\)",
+            r'  running command "git checkout 8456f86"',
+            r"  \(in testrepository\)",
+            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+            r"  \(in .*/tmp.*\)",
         ])
         run_check()
 
@@ -2004,7 +2014,7 @@ def test_is_sha256_checksum(self):
             True,
             12345,
             '',
-            (a_sha256_checksum, ),
+            (a_sha256_checksum,),
             [],
         ]:
             self.assertFalse(ft.is_sha256_checksum(not_a_sha256_checksum))
@@ -2066,7 +2076,6 @@ def test_fake_vsc(self):
         self.assertTrue(pkgutil.__file__.endswith('/test_fake_vsc/pkgutil.py'))
 
 
-
 def suite():
     """ returns all the testcases in this module """
     return TestLoaderFiltered().loadTestsFromTestCase(FileToolsTest, sys.argv[1:])