Repository: apify/crawlee-python
File: src/crawlee/storages/_key_value_store.py (Python)

Code before:
        content_type: str | None = None,
    ) -> None:
        """Set a value in the KVS.

        Args:
            key: Key of the record to set.
            value: Value to set. If `None`, the record is deleted.
            content_type: Content type of the record.
        """
        if value is None:
            return await self._resource_client.delete_record(key)

        return await self._resource_client.set_record(key, value, content_type)

    async def get_public_url(self, key: str) -> str:
        """Get the public URL for the given key.

        Args:
            key: Key of the record for which URL is required.

        Returns:
            The public URL for the given key.
        """
        return await self._resource_client.get_public_url(key)

    async def get_auto_saved_value(self, key: str, default_value: dict[str, Any] | None = None) -> dict[str, Any]:
        """Gets a value from store that will be automatically saved on changes.

        Args:
            key: Key of the record, to store the value.
            default_value: value to be used if the record does not exist yet. Should be a dictionary

        Returns:
            Returns the value of the key
        """
        default_value = {} if default_value is None else default_value

        if key in self._cache:
            return self._cache[key]

        value = await self.get_value(key, default_value)

        self._cache[key] = value

        self._ensure_persist_event()

        return cast(dict[str, Any], value)

    def _ensure_persist_event(self) -> None:
        """Setup persist state event handling if not already done."""
        if self._persist_state_event_started:

Review comment (suggestion):
Currently, crawlee-js does this in two separate ways. What you ported over is the "regular" one. Apart from that, we have adaptive crawling, where `useState` bypasses `get_auto_saved_value`. See https://github.com/apify/crawlee/blob/master/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts for details. 

The general idea behind this is that we don't want to "save" the changed state before we know that the request handler succeeded.

We should probably discuss what would be the best way forward. Try to give the javascript adaptive crawler a glance, but if it doesn't make any sense, just say so so that we don't waste any time. Then we'll talk more :slightly_smiling_face: 

Code after:

    async def set_value(
        self,
        key: str,
        value: Any,
        content_type: str | None = None,
    ) -> None:
        """Set a value in the KVS.

        Args:
            key: Key of the record to set.
            value: Value to set. If `None`, the record is deleted.
            content_type: Content type of the record.
        """
        if value is None:
            return await self._resource_client.delete_record(key)

        return await self._resource_client.set_record(key, value, content_type)

    async def get_public_url(self, key: str) -> str:
        """Get the public URL for the given key.

        Args:
            key: Key of the record for which URL is required.

        Returns:
            The public URL for the given key.
        """
        return await self._resource_client.get_public_url(key)

    async def get_auto_saved_value(
        self, key: str, default_value: dict[str, JsonSerializable] | None = None
    ) -> dict[str, JsonSerializable]:
        """Gets a value from KVS that will be automatically saved on changes.

        Args:
            key: Key of the record, to store the value.
            default_value: Value to be used if the record does not exist yet. Should be a dictionary.

        Returns:
            Returns the value of the key.
        """
        default_value = {} if default_value is None else default_value

        if key in self._cache:
            return self._cache[key]

        value = await self.get_value(key, default_value)

        if not isinstance(value, dict):
            raise TypeError(