Repository: Chia-Network/chia-blockchain
File: chia/data_layer/data_store.py (Python)

Code before:

            found = await anyio.to_thread.run_sync(
                delta_reader.collect_and_return_from_merkle_blobs,
                jobs,
                missing_hashes,
            )
            async with self.db_wrapper.writer() as writer:
                await writer.executemany(
                    """
                    INSERT
                    OR IGNORE INTO nodes(store_id, hash, root_hash, generation, idx)
                    VALUES (?, ?, ?, ?, ?)
                    """,
                    (
                        (store_id, hash, root_hash, generations_by_root_hash[root_hash], index.raw)
                        for root_hash, map in found
                        for hash, index in map.items()
                    ),
                )

            missing_hashes = delta_reader.get_missing_hashes(root_hash)

            log.info(f"Missing hashes: added old hashes from generation {current_generation}")

    async def read_from_file(
        self, filename: Path, store_id: bytes32, writer: aiosqlite.Connection
    ) -> tuple[dict[bytes32, tuple[bytes32, bytes32]], dict[bytes32, tuple[KeyId, ValueId]]]:
        internal_nodes: dict[bytes32, tuple[bytes32, bytes32]] = {}
        terminal_nodes: dict[bytes32, tuple[KeyId, ValueId]] = {}

        with open(filename, "rb") as reader:
            while True:
                chunk = b""
                while len(chunk) < 4:
                    size_to_read = 4 - len(chunk)
                    cur_chunk = reader.read(size_to_read)
                    if cur_chunk is None or cur_chunk == b"":
                        if size_to_read < 4:
                            raise Exception("Incomplete read of length.")
                        break
                    chunk += cur_chunk
                if chunk == b"":
                    break

                size = int.from_bytes(chunk, byteorder="big")
                serialize_nodes_bytes = b""
                while len(serialize_nodes_bytes) < size:
                    size_to_read = size - len(serialize_nodes_bytes)
                    cur_chunk = reader.read(size_to_read)
                    if cur_chunk is None or cur_chunk == b"":
                        raise Exception("Incomplete read of blob.")

Review comment (performance):
in most places we only pass a writer when it is critical for performance.  that doesn't seem like it would be the case here.  the nesting should just naturally work.  is there another reason to be passing the writer in?

Code after:
                jobs,
                missing_hashes,
            )
            async with self.db_wrapper.writer() as writer:
                await writer.executemany(
                    """
                    INSERT
                    OR IGNORE INTO nodes(store_id, hash, root_hash, generation, idx)
                    VALUES (?, ?, ?, ?, ?)
                    """,
                    (
                        (store_id, hash, root_hash, generations_by_root_hash[root_hash], index.raw)
                        for root_hash, map in found
                        for hash, index in map.items()
                    ),
                )

            missing_hashes = delta_reader.get_missing_hashes(root_hash)

            log.info(f"Missing hashes: added old hashes from generation {current_generation}")

    async def read_from_file(
        self, filename: Path, store_id: bytes32
    ) -> tuple[dict[bytes32, tuple[bytes32, bytes32]], dict[bytes32, tuple[KeyId, ValueId]]]:
        internal_nodes: dict[bytes32, tuple[bytes32, bytes32]] = {}
        terminal_nodes: dict[bytes32, tuple[KeyId, ValueId]] = {}

        with open(filename, "rb") as reader:
            async with self.db_wrapper.writer() as writer:
                while True:
                    chunk = b""
                    while len(chunk) < 4:
                        size_to_read = 4 - len(chunk)
                        cur_chunk = reader.read(size_to_read)
                        if cur_chunk is None or cur_chunk == b"":
                            if size_to_read < 4:
                                raise Exception("Incomplete read of length.")
                            break
                        chunk += cur_chunk
                    if chunk == b"":
                        break

                    size = int.from_bytes(chunk, byteorder="big")
                    serialize_nodes_bytes = b""
                    while len(serialize_nodes_bytes) < size:
                        size_to_read = size - len(serialize_nodes_bytes)
                        cur_chunk = reader.read(size_to_read)
                        if cur_chunk is None or cur_chunk == b"":
                            raise Exception("Incomplete read of blob.")
                        serialize_nodes_bytes += cur_chunk
                    serialized_node = SerializedNode.from_bytes(serialize_nodes_bytes)