Repository: apify/crawlee-python
File: src/crawlee/storage_clients/_memory/_creation_management.py (Python)

Code before:

logger = getLogger(__name__)


async def persist_metadata_if_enabled(*, data: dict, entity_directory: str, write_metadata: bool) -> None:
    """Updates or writes metadata to a specified directory.

    The function writes a given metadata dictionary to a JSON file within a specified directory.
    The writing process is skipped if `write_metadata` is False. Before writing, it ensures that
    the target directory exists, creating it if necessary.

    Args:
        data: A dictionary containing metadata to be written.
        entity_directory: The directory path where the metadata file should be stored.
        write_metadata: A boolean flag indicating whether the metadata should be written to file.
    """
    # Skip metadata write; ensure directory exists first
    if not write_metadata:
        return

    # Ensure the directory for the entity exists
    await asyncio.to_thread(os.makedirs, entity_directory, exist_ok=True)

    # Write the metadata to the file
    file_path = os.path.join(entity_directory, METADATA_FILENAME)
    mode = 'r+b' if os.path.exists(file_path) else 'wb'
    f = await asyncio.to_thread(open, file_path, mode=mode)
    try:
        s = await json_dumps(data)
        await asyncio.to_thread(f.write, s.encode('utf-8'))
    finally:
        await asyncio.to_thread(f.close)


def find_or_create_client_by_id_or_name_inner(
    resource_client_class: type[TResourceClient],
    memory_storage_client: MemoryStorageClient,
    id: str | None = None,
    name: str | None = None,
) -> TResourceClient | None:
    """Locates or creates a new storage client based on the given ID or name.

    This method attempts to find a storage client in the memory cache first. If not found,
    it tries to locate a storage directory by name. If still not found, it searches through
    storage directories for a matching ID or name in their metadata. If none exists, and the
    specified ID is 'default', it checks for a default storage directory. If a storage client
    is found or created, it is added to the memory cache. If no storage client can be located or
    created, the method returns None.

    Args:
        resource_client_class: The class of the resource client.

Review comment (suggestion):
If the file exists we can still open it with "wb" mode.If we open it with "r+b" mode, then we are not overwriting the whole file with new content but just changing the file starting from the beginning of the file. I doubt that is what we want.

Imagine file with content `b"abc"`
and you want to change it
```
with open(path, 'r+b') as f:
    f.write(b"x") 
``` 
->` b"xbc"`


```
with open(path, 'wb') as f:
    f.write(b"x") 
``` 
-> `b"x"`

Maybe I have missed the point here, but so far it does not seem right to me.

Code after:

logger = getLogger(__name__)


async def persist_metadata_if_enabled(*, data: dict, entity_directory: str, write_metadata: bool) -> None:
    """Updates or writes metadata to a specified directory.

    The function writes a given metadata dictionary to a JSON file within a specified directory.
    The writing process is skipped if `write_metadata` is False. Before writing, it ensures that
    the target directory exists, creating it if necessary.

    Args:
        data: A dictionary containing metadata to be written.
        entity_directory: The directory path where the metadata file should be stored.
        write_metadata: A boolean flag indicating whether the metadata should be written to file.
    """
    # Skip metadata write; ensure directory exists first
    if not write_metadata:
        return

    # Ensure the directory for the entity exists
    await asyncio.to_thread(os.makedirs, entity_directory, exist_ok=True)

    # Write the metadata to the file
    file_path = os.path.join(entity_directory, METADATA_FILENAME)
    f = await asyncio.to_thread(open, file_path, mode='wb')
    try:
        s = await json_dumps(data)
        await asyncio.to_thread(f.write, s.encode('utf-8'))
    finally:
        await asyncio.to_thread(f.close)


def find_or_create_client_by_id_or_name_inner(
    resource_client_class: type[TResourceClient],
    memory_storage_client: MemoryStorageClient,
    id: str | None = None,
    name: str | None = None,
) -> TResourceClient | None:
    """Locates or creates a new storage client based on the given ID or name.

    This method attempts to find a storage client in the memory cache first. If not found,
    it tries to locate a storage directory by name. If still not found, it searches through
    storage directories for a matching ID or name in their metadata. If none exists, and the
    specified ID is 'default', it checks for a default storage directory. If a storage client
    is found or created, it is added to the memory cache. If no storage client can be located or
    created, the method returns None.

    Args:
        resource_client_class: The class of the resource client.
        memory_storage_client: The memory storage client used to store and retrieve storage clients.