| """Utilities to efficiently compute the SHA 256 hash of a bunch of bytes.""" |
|
|
| from typing import BinaryIO, Optional |
|
|
| from .insecure_hashlib import sha1, sha256 |
|
|
|
|
| def sha_fileobj(fileobj: BinaryIO, chunk_size: Optional[int] = None) -> bytes: |
| """ |
| Computes the sha256 hash of the given file object, by chunks of size `chunk_size`. |
| |
| Args: |
| fileobj (file-like object): |
| The File object to compute sha256 for, typically obtained with `open(path, "rb")` |
| chunk_size (`int`, *optional*): |
| The number of bytes to read from `fileobj` at once, defaults to 1MB. |
| |
| Returns: |
| `bytes`: `fileobj`'s sha256 hash as bytes |
| """ |
| chunk_size = chunk_size if chunk_size is not None else 1024 * 1024 |
|
|
| sha = sha256() |
| while True: |
| chunk = fileobj.read(chunk_size) |
| sha.update(chunk) |
| if not chunk: |
| break |
| return sha.digest() |
|
|
|
|
| def git_hash(data: bytes) -> str: |
| """ |
| Computes the git-sha1 hash of the given bytes, using the same algorithm as git. |
| |
| This is equivalent to running `git hash-object`. See https://git-scm.com/docs/git-hash-object |
| for more details. |
| |
| Note: this method is valid for regular files. For LFS files, the proper git hash is supposed to be computed on the |
| pointer file content, not the actual file content. However, for simplicity, we directly compare the sha256 of |
| the LFS file content when we want to compare LFS files. |
| |
| Args: |
| data (`bytes`): |
| The data to compute the git-hash for. |
| |
| Returns: |
| `str`: the git-hash of `data` as an hexadecimal string. |
| |
| Example: |
| ```python |
| >>> from huggingface_hub.utils.sha import git_hash |
| >>> git_hash(b"Hello, World!") |
| 'b45ef6fec89518d314f546fd6c3025367b721684' |
| ``` |
| """ |
| |
| |
| sha = sha1() |
| sha.update(b"blob ") |
| sha.update(str(len(data)).encode()) |
| sha.update(b"\0") |
| sha.update(data) |
| return sha.hexdigest() |
|
|
