Repository: pulp/pulpcore

Issue:
The export process spends a significant amount of time on hashing
**Describe the bug**

The export process hashes chunks, but then also creates a "global hash" of all of the files processed in-order.  This means that multiple sha256 checksums are being calculated in parallel, across a very large amount of data (potentially terabytes).

At least on small exports, this adds up to a significant portion of the total import time.  

What we could do instead is checksum the chunks once, and then use a checksum-of-checksums for the global checksum, as discussed here.

https://github.com/pulp/pulpcore/pull/4435#issuecomment-1722110281

**To Reproduce**

Perform an export with profiling enabled, observe the profile, see that ~1/3rd of it is spent performing sha256 checksums.  We can reduce that by half.



Discussion:
@mdellweg @ggainey does any of you know why in case of chunked export we still provide global checksum https://github.com/dralley/pulpcore/blob/main/pulpcore/app/tasks/export.py#L502? As @dralley  points out apparently it is expensive to perform check-summing.
To me it feels redundant to why we'd provide global checksum in case of chunked export as long as we provide in the json  file mapping between each chunk and its checksim which on destination host transfer can be verified https://github.com/dralley/pulpcore/blob/main/pulpcore/app/tasks/export.py#L476
It appears that we reconstruct the whole file using its constituent parts [here](https://github.com/pulp/pulpcore/blob/main/pulpcore/app/tasks/importer.py#L386)

So the chunk checksums serve to provide an early warning that something is wrong, and the total file checksum serves to ensure that the reconstruction occurred correctly.  We also delete chunks after adding them to the reconstructed file, to avoid requiring twice as much disk space, so you can't "go back" easily.

We can at least parallelize the checksumming of the chunks easily so I'm going to do that.  But I'm not certain we can fully get rid of the global checksum without replacing it somehow.

Since this is not security sensitive we could use a faster checksum such as crc32 or crc64 though, especially if we keep using sha256 on chunks.
> We also delete chunks after adding them to the reconstructed file, to avoid requiring twice as much disk space, so you can't "go back" easily.

Is there any way, we can concatenate the chunks on the fly without changing anything on disk (think import from ro filesystem)? Like running cat and piping the result into django-import?
I don't think so, I don't expect there's any way to work with multiple files as if they were one in that way.  Also we don't necessarily stream in one direction, it's treated as a tarfile, which has a header, and we work with the tarfile such that I believe it goes back to reference the header.

Resolving patch:
diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py
--- a/pulpcore/app/tasks/export.py
+++ b/pulpcore/app/tasks/export.py
@@ -1,4 +1,3 @@
-import hashlib
 import json
 import logging
 import os
@@ -28,7 +27,7 @@
 from pulpcore.app.models.content import Artifact, ContentArtifact
 from pulpcore.app.serializers import PulpExportSerializer
 
-from pulpcore.app.util import compute_file_hash, get_version_from_model
+from pulpcore.app.util import compute_file_hash, get_version_from_model, Crc32Hasher
 from pulpcore.app.importexport import (
     export_versions,
     export_artifacts,
@@ -386,6 +385,8 @@ def pulp_export(exporter_pk, params):
     the_export.validated_start_versions = serializer.validated_data.get("start_versions", None)
     the_export.validated_chunk_size = serializer.validated_data.get("chunk_size", None)
 
+    hasher = Crc32Hasher
+    checksum_type = "crc32"
     try:
         the_export.task = Task.current()
 
@@ -425,10 +426,10 @@ def pulp_export(exporter_pk, params):
                         os.remove(pathname)
                     raise
             # compute the hashes
-            global_hash = hashlib.sha256()
+            global_hash = hasher()
             paths = sorted([str(Path(p)) for p in glob(tarfile_fp + ".*")])
             for a_file in paths:
-                a_hash = compute_file_hash(a_file, cumulative_hash=global_hash)
+                a_hash = compute_file_hash(a_file, hasher=hasher(), cumulative_hash=global_hash)
                 rslts[a_file] = a_hash
             tarfile_hash = global_hash.hexdigest()
 
@@ -444,7 +445,7 @@ def pulp_export(exporter_pk, params):
                     os.remove(tarfile_fp)
                 raise
             # compute the hash
-            tarfile_hash = compute_file_hash(tarfile_fp)
+            tarfile_hash = compute_file_hash(tarfile_fp, hasher=hasher())
             rslts[tarfile_fp] = tarfile_hash
 
         # store the outputfile/hash info
@@ -462,6 +463,7 @@ def pulp_export(exporter_pk, params):
                     "chunk_size": chunk_size,
                     "file": os.path.basename(tarfile_fp),
                     "global_hash": tarfile_hash,
+                    "checksum_type": checksum_type,
                 },
                 "files": {},
             }
diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py
--- a/pulpcore/app/tasks/importer.py
+++ b/pulpcore/app/tasks/importer.py
@@ -35,7 +35,7 @@
     ContentArtifactResource,
     RepositoryResource,
 )
-from pulpcore.app.util import compute_file_hash
+from pulpcore.app.util import compute_file_hash, Crc32Hasher
 from pulpcore.constants import TASK_STATES
 from pulpcore.tasking.tasks import dispatch
 
@@ -356,18 +356,23 @@ def validate_toc(toc_filename):
                 )
 
             errs = []
-            # validate the sha256 of the toc-entries
+
+            def verify_chunk_hash(chunk_path, expected_digest):
+                actual_digest = compute_file_hash(chunk_path, hasher=Crc32Hasher())
+                if actual_digest != expected_digest:
+                    err_str = "File {} expected checksum : {}, computed checksum : {}".format(
+                        chunk, expected_digest, actual_digest
+                    )
+                    errs.append(err_str)
+
+            # validate the checksum of the toc-entries
             # gather errors for reporting at the end
             chunks = sorted(the_toc["files"].keys())
             data = dict(message="Validating Chunks", code="validate.chunks", total=len(chunks))
             with ProgressReport(**data) as pb:
                 for chunk in pb.iter(chunks):
-                    a_hash = compute_file_hash(os.path.join(base_dir, chunk))
-                    if not a_hash == the_toc["files"][chunk]:
-                        err_str = "File {} expected checksum : {}, computed checksum : {}".format(
-                            chunk, the_toc["files"][chunk], a_hash
-                        )
-                        errs.append(err_str)
+                    chunk_path = os.path.join(base_dir, chunk)
+                    verify_chunk_hash(chunk_path, the_toc["files"][chunk])
 
             # if there are any errors, report and fail
             if errs:
@@ -416,7 +421,7 @@ def reassemble(the_toc, toc_dir, result_file):
                         exc_info=True,
                     )
 
-        combined_hash = compute_file_hash(result_file)
+        combined_hash = compute_file_hash(result_file, hasher=Crc32Hasher())
         if combined_hash != the_toc["meta"]["global_hash"]:
             raise ValidationError(
                 _("Mismatch between combined .tar checksum [{}] and originating [{}]).").format(
diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py
--- a/pulpcore/app/util.py
+++ b/pulpcore/app/util.py
@@ -1,4 +1,5 @@
 import hashlib
+import zlib
 from functools import lru_cache
 from gettext import gettext as _
 import os
@@ -309,6 +310,22 @@ def compute_file_hash(filename, hasher=None, cumulative_hash=None, blocksize=819
         return hasher.hexdigest()
 
 
+class Crc32Hasher:
+    """Wrapper to make the CRC32 implementation act like a standard hashlib hasher"""
+
+    def __init__(self):
+        self.hashval = 0
+
+    def update(self, data):
+        self.hashval = zlib.crc32(data, self.hashval)
+
+    def digest(self):
+        return str(self.hashval)
+
+    def hexdigest(self):
+        return hex(self.hashval)[2:]
+
+
 def configure_analytics():
     task_name = "pulpcore.app.tasks.analytics.post_analytics"
     dispatch_interval = timedelta(days=1)


Test patch: