Repository: apify/crawlee-python
File: src/crawlee/crawlers/_basic/_basic_crawler.py (Python)

Code before:
        except RequestCollisionError as request_error:
            context.request.no_retry = True
            await self._handle_request_error(context, request_error)

        except RequestHandlerError as primary_error:
            primary_error = cast(
                'RequestHandlerError[TCrawlingContext]', primary_error
            )  # valid thanks to ContextPipeline

            self._logger.debug(
                'An exception occurred in the user-defined request handler',
                exc_info=primary_error.wrapped_exception,
            )
            await self._handle_request_error(primary_error.crawling_context, primary_error.wrapped_exception)

        except SessionError as session_error:
            if not context.session:
                raise RuntimeError('SessionError raised in a crawling context without a session') from session_error

            if self._error_handler:
                await self._error_handler(context, session_error)

            if self._should_retry_request(context, session_error):
                self._logger.warning('Encountered a session error, rotating session and retrying')

                context.session.retire()

                if context.request.session_rotation_count is None:
                    context.request.session_rotation_count = 0
                context.request.session_rotation_count += 1

                await request_manager.reclaim_request(request)
                self._statistics.error_tracker_retry.add(session_error)
            else:
                self._logger.exception('Request failed and reached maximum retries', exc_info=session_error)

                await wait_for(
                    lambda: request_manager.mark_request_as_handled(context.request),
                    timeout=self._internal_timeout,
                    timeout_message='Marking request as handled timed out after '
                    f'{self._internal_timeout.total_seconds()} seconds',
                    logger=self._logger,
                    max_retries=3,
                )

                self._statistics.record_request_processing_failure(statistics_id)
                self._statistics.error_tracker.add(session_error)

        except ContextPipelineInterruptedError as interrupted_error:
            self._logger.debug('The context pipeline was interrupted', exc_info=interrupted_error)


Review comment (bug):
So in case of a collision, the request will be aborted, right? Unless the user re-enqueues it in the `failed_request_handler`, which is kinda cumbersome. 

I imagine it can happen quite frequently that the session will get blocked and rotated out. I understand that silently using a new, different session might be confusing (but it might also be a viable option). Nevertheless, there should be an easier way to handle this in user code and it should be described in the documentation.

Code after:
            try:
                await self._run_request_handler(context=context)
            except asyncio.TimeoutError as e:
                raise RequestHandlerError(e, context) from e

            await self._commit_request_handler_result(context)
            await wait_for(
                lambda: request_manager.mark_request_as_handled(context.request),
                timeout=self._internal_timeout,
                timeout_message='Marking request as handled timed out after '
                f'{self._internal_timeout.total_seconds()} seconds',
                logger=self._logger,
                max_retries=3,
            )

            request.state = RequestState.DONE

            if context.session and context.session.is_usable:
                context.session.mark_good()

            self._statistics.record_request_processing_finish(statistics_id)

        except RequestCollisionError as request_error:
            context.request.no_retry = True
            await self._handle_request_error(context, request_error)

        except RequestHandlerError as primary_error:
            primary_error = cast(
                'RequestHandlerError[TCrawlingContext]', primary_error
            )  # valid thanks to ContextPipeline

            self._logger.debug(
                'An exception occurred in the user-defined request handler',
                exc_info=primary_error.wrapped_exception,
            )
            await self._handle_request_error(primary_error.crawling_context, primary_error.wrapped_exception)

        except SessionError as session_error:
            if not context.session:
                raise RuntimeError('SessionError raised in a crawling context without a session') from session_error

            if self._error_handler:
                await self._error_handler(context, session_error)

            if self._should_retry_request(context, session_error):
                self._logger.warning('Encountered a session error, rotating session and retrying')

                context.session.retire()

                if context.request.session_rotation_count is None:
                    context.request.session_rotation_count = 0