Repository: mesonbuild/meson

Issue:
Should download archive to disk and then do sha256sum
some archive is very big, example sdk from bsp vendor, some is 1G, 2G size, if download  archive to memory and do sha256sum ,  in this case, it is terrible.

```diff
diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py
index ac84d0e4..147e3ea1 100644
--- a/mesonbuild/wrap/wrap.py
+++ b/mesonbuild/wrap/wrap.py
@@ -228,7 +228,7 @@ class Resolver:
                 subprocess.check_call(['hg', 'checkout', revno],
                                       cwd=checkoutdir)

-    def get_data(self, url):
+    def get_data(self, url, ofname):
         blocksize = 10 * 1024
         if url.startswith('https://wrapdb.mesonbuild.com'):
             resp = open_wrapdburl(url)
@@ -241,26 +241,29 @@ class Resolver:
                 dlsize = None
             if dlsize is None:
                 print('Downloading file of unknown size.')
-                return resp.read()
+                with open(ofname, "wb") as f:
+                    f.write(resp.read())
+                return
+
             print('Download size:', dlsize)
             print('Downloading: ', end='')
             sys.stdout.flush()
             printed_dots = 0
-            blocks = []
             downloaded = 0
-            while True:
-                block = resp.read(blocksize)
-                if block == b'':
-                    break
-                downloaded += len(block)
-                blocks.append(block)
-                ratio = int(downloaded / dlsize * 10)
-                while printed_dots < ratio:
-                    print('.', end='')
-                    sys.stdout.flush()
-                    printed_dots += 1
-            print('')
-        return b''.join(blocks)
+
+            with open(ofname, "wb") as f:
+                while True:
+                    wlen = f.write(resp.read(blocksize))
+                    downloaded += wlen
+                    if downloaded == dlsize:
+                        break
+                    ratio = int(downloaded / dlsize * 10)
+                    while printed_dots < ratio:
+                        print('.', end='')
+                        sys.stdout.flush()
+                        printed_dots += 1
+                print('')
+        return

     def get_hash(self, data):
         h = hashlib.sha256()
@@ -268,6 +271,13 @@ class Resolver:
         hashvalue = h.hexdigest()
         return hashvalue

+    def file_hash(self, filename):
+        h = hashlib.sha256()
+        with open(filename, 'rb', buffering=0) as f:
+            for b in iter(lambda : f.read(128*1024), b''):
+                h.update(b)
+        return h.hexdigest()
+
     def download(self, p, packagename):
         ofname = os.path.join(self.cachedir, p.get('source_filename'))
         if os.path.exists(ofname):
@@ -275,24 +285,23 @@ class Resolver:
             return
         srcurl = p.get('source_url')
         mlog.log('Downloading', mlog.bold(packagename), 'from', mlog.bold(srcurl))
-        srcdata = self.get_data(srcurl)
-        dhash = self.get_hash(srcdata)
+        self.get_data(srcurl, ofname)
+        dhash = self.file_hash(ofname)
         expected = p.get('source_hash')
         if dhash != expected:
+            os.unlink(ofname)
             raise RuntimeError('Incorrect hash for source %s:\n %s expected\n %s actual.' % (packagename, expected, dhash))
-        with open(ofname, 'wb') as f:
-            f.write(srcdata)
         if p.has_patch():
             purl = p.get('patch_url')
+            filename = os.path.join(self.cachedir, p.get('patch_filename'))
             mlog.log('Downloading patch from', mlog.bold(purl))
-            pdata = self.get_data(purl)
-            phash = self.get_hash(pdata)
+            self.get_data(purl, filename)
+            phash = self.file_hash(filename)
             expected = p.get('patch_hash')
             if phash != expected:
+                os.unlink(filename)
                 raise RuntimeError('Incorrect hash for patch %s:\n %s expected\n %s actual' % (packagename, expected, phash))
             filename = os.path.join(self.cachedir, p.get('patch_filename'))
-            with open(filename, 'wb') as f:
-                f.write(pdata)
         else:
             mlog.log('Package does not require patch.')


```


Discussion:

Resolving patch:
diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py
--- a/mesonbuild/wrap/wrap.py
+++ b/mesonbuild/wrap/wrap.py
@@ -14,7 +14,7 @@
 
 from .. import mlog
 import contextlib
-import urllib.request, os, hashlib, shutil
+import urllib.request, os, hashlib, shutil, tempfile, stat
 import subprocess
 import sys
 from pathlib import Path
@@ -230,6 +230,8 @@ def get_hg(self, p):
 
     def get_data(self, url):
         blocksize = 10 * 1024
+        h = hashlib.sha256()
+        tmpfile = tempfile.NamedTemporaryFile(mode='wb', dir=self.cachedir, delete=False)
         if url.startswith('https://wrapdb.mesonbuild.com'):
             resp = open_wrapdburl(url)
         else:
@@ -241,26 +243,34 @@ def get_data(self, url):
                 dlsize = None
             if dlsize is None:
                 print('Downloading file of unknown size.')
-                return resp.read()
+                while True:
+                    block = resp.read(blocksize)
+                    if block == b'':
+                        break
+                    h.update(block)
+                    tmpfile.write(block)
+                hashvalue = h.hexdigest()
+                return hashvalue, tmpfile.name
             print('Download size:', dlsize)
             print('Downloading: ', end='')
             sys.stdout.flush()
             printed_dots = 0
-            blocks = []
             downloaded = 0
             while True:
                 block = resp.read(blocksize)
                 if block == b'':
                     break
                 downloaded += len(block)
-                blocks.append(block)
+                h.update(block)
+                tmpfile.write(block)
                 ratio = int(downloaded / dlsize * 10)
                 while printed_dots < ratio:
                     print('.', end='')
                     sys.stdout.flush()
                     printed_dots += 1
             print('')
-        return b''.join(blocks)
+            hashvalue = h.hexdigest()
+        return hashvalue, tmpfile.name
 
     def get_hash(self, data):
         h = hashlib.sha256()
@@ -272,30 +282,47 @@ def download(self, p, packagename):
         ofname = os.path.join(self.cachedir, p.get('source_filename'))
         if os.path.exists(ofname):
             mlog.log('Using', mlog.bold(packagename), 'from cache.')
-            return
-        srcurl = p.get('source_url')
-        mlog.log('Downloading', mlog.bold(packagename), 'from', mlog.bold(srcurl))
-        srcdata = self.get_data(srcurl)
-        dhash = self.get_hash(srcdata)
-        expected = p.get('source_hash')
-        if dhash != expected:
-            raise RuntimeError('Incorrect hash for source %s:\n %s expected\n %s actual.' % (packagename, expected, dhash))
-        with open(ofname, 'wb') as f:
-            f.write(srcdata)
+        else:
+            srcurl = p.get('source_url')
+            mlog.log('Downloading', mlog.bold(packagename), 'from', mlog.bold(srcurl))
+            dhash, tmpfile = self.get_data(srcurl)
+            expected = p.get('source_hash')
+            if dhash != expected:
+                os.remove(tmpfile)
+                raise RuntimeError('Incorrect hash for source %s:\n %s expected\n %s actual.' % (packagename, expected, dhash))
+            os.rename(tmpfile, ofname)
         if p.has_patch():
             purl = p.get('patch_url')
             mlog.log('Downloading patch from', mlog.bold(purl))
-            pdata = self.get_data(purl)
-            phash = self.get_hash(pdata)
+            phash, tmpfile = self.get_data(purl)
             expected = p.get('patch_hash')
             if phash != expected:
+                os.remove(tmpfile)
                 raise RuntimeError('Incorrect hash for patch %s:\n %s expected\n %s actual' % (packagename, expected, phash))
             filename = os.path.join(self.cachedir, p.get('patch_filename'))
-            with open(filename, 'wb') as f:
-                f.write(pdata)
+            os.rename(tmpfile, filename)
         else:
             mlog.log('Package does not require patch.')
 
+    def copy_tree(self, root_src_dir, root_dst_dir):
+        """
+        Copy directory tree. Overwrites also read only files.
+        """
+        for src_dir, dirs, files in os.walk(root_src_dir):
+            dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
+            if not os.path.exists(dst_dir):
+                os.makedirs(dst_dir)
+            for file_ in files:
+                src_file = os.path.join(src_dir, file_)
+                dst_file = os.path.join(dst_dir, file_)
+                if os.path.exists(dst_file):
+                    try:
+                        os.remove(dst_file)
+                    except PermissionError as exc:
+                        os.chmod(dst_file, stat.S_IWUSR)
+                        os.remove(dst_file)
+                shutil.copy2(src_file, dst_dir)
+
     def extract_package(self, package):
         if sys.version_info < (3, 5):
             try:
@@ -322,4 +349,9 @@ def extract_package(self, package):
             pass
         shutil.unpack_archive(os.path.join(self.cachedir, package.get('source_filename')), extract_dir)
         if package.has_patch():
-            shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), self.subdir_root)
+            try:
+                shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), self.subdir_root)
+            except Exception:
+                with tempfile.TemporaryDirectory() as workdir:
+                    shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), workdir)
+                    self.copy_tree(workdir, self.subdir_root)


Test patch: