Repository: apify/crawlee-python
File: tests/unit/request_loaders/test_request_list.py (Python)

Code before:
    spy = iter(['1', '2', '3'])
    request_list_2 = RequestList(
        spy,
        persist_requests_key=persist_requests_key,
        persist_state_key=persist_state_key,
    )

    # Should be able to fetch requests from persisted data and continue where the previous instance left off
    next_request = await request_list_2.fetch_next_request()
    assert next_request is not None
    assert next_request.url == 'https://restore2.placeholder.com'
    await request_list_2.mark_request_as_handled(next_request)

    # Make sure that the second instance did not consume the input iterator
    assert len(list(spy)) == 3


async def test_persist_requests_key_only_persists_once() -> None:
    """Test that requests are only persisted once, even with multiple RequestList instances."""
    persist_key = 'test_requests_once'
    urls = ['https://once1.placeholder.com', 'https://once2.placeholder.com']

    # Create first request list
    request_list_1 = RequestList(urls, persist_requests_key=persist_key)
    await request_list_1.fetch_next_request()  # Trigger persistence

    # Get initial persisted data
    kvs = await KeyValueStore.open()
    initial_data = await kvs.get_value(persist_key)
    assert initial_data is not None

    # Create second request list with different data
    different_urls = ['https://different.placeholder.com']
    request_list_2 = RequestList(different_urls, persist_requests_key=persist_key)
    await request_list_2.fetch_next_request()  # Should use persisted data, not new data

    # Verify the persisted data hasn't changed
    current_data = await kvs.get_value(persist_key)
    assert current_data == initial_data

    # The request should come from the original persisted data, not the new iterator
    fetched_request = await request_list_2.fetch_next_request()
    assert fetched_request is not None
    assert fetched_request.url == 'https://once2.placeholder.com'  # From original data


Review comment (bug):
I tried extending this test by extra request and it failed.

So having these URLs

`urls = ['https://restore1.placeholder.com', 'https://restore2.placeholder.com', 'https://restore3.placeholder.com']`

and adding this

```
    third_request = await request_list_2.fetch_next_request()
    assert third_request is not None
    assert third_request.url == 'https://restore3.placeholder.com'
    await request_list_2.mark_request_as_handled(third_request)
```

Code after:
    await request_list_1._state.persist_state()

    # Create second request list with same persist key (simulating restart)
    request_list_2 = RequestList(
        urls,
        persist_state_key=persist_state_key,
    )

    # Should be able to continue where the previous instance left off
    next_request = await request_list_2.fetch_next_request()
    assert next_request is not None
    assert next_request.url == 'https://restore2.placeholder.com'
    await request_list_2.mark_request_as_handled(next_request)

    next_request = await request_list_2.fetch_next_request()
    assert next_request is not None
    assert next_request.url == 'https://restore3.placeholder.com'
    await request_list_2.mark_request_as_handled(next_request)

    next_request = await request_list_2.fetch_next_request()
    assert next_request is not None
    assert next_request.url == 'https://restore4.placeholder.com'
    await request_list_2.mark_request_as_handled(next_request)


async def test_requests_and_state_restoration() -> None:
    """Test that persisted request data and processing state is properly restored on subsequent RequestList creation."""
    persist_requests_key = 'test_requests_restoration'
    persist_state_key = 'test_state_restoration'
    urls = [
        'https://restore1.placeholder.com',
        'https://restore2.placeholder.com',
        'https://restore3.placeholder.com',
    ]

    # Create first request list and process one request
    request_list_1 = RequestList(
        urls,
        persist_requests_key=persist_requests_key,
        persist_state_key=persist_state_key,
    )

    first_request = await request_list_1.fetch_next_request()
    assert first_request is not None
    assert first_request.url == 'https://restore1.placeholder.com'
    await request_list_1.mark_request_as_handled(first_request)
    await request_list_1._state.persist_state()

    # Create second request list with same persist key (simulating restart)
    spy = iter(['1', '2', '3'])
    request_list_2 = RequestList(