Repository: rq/rq
Pull request #2208: Update the StartedJobRegistry to use correct job_id extracting
Author: login: OlegZv
Created: 2025-03-15T18:54:28Z | Merged: 2025-03-20T10:43:55Z

* Added `parse_job_id` method. BaseRegistry would return the id as_text, but StartedJobRegistry would parse the job_id from `job_id:execution` format.
* StartedJobRegistry.cleanup() now properly saves the job result into job result (using the _handle_failure)
* Added an option for the `Queue.enqueue_dependents` to prevent refreshing the job status, if the job status has been updated in the pipeline)
* Added deprecation notices to avoid adding/removing a job to the StartedJobRegistry (only Executions are allowed)
* Added a test where the job that is still executing would not be cleaned up, while in the started queue (test_no_cleanup_while_in_started_queue)
* Updated tests to exercise the StartedJobRegistry separately from the BaseRegistry.
* Added tests for a few warnings


Diff:
diff --git a/rq/executions.py b/rq/executions.py
index bf2a7dfe2..bf3cba9d4 100644
--- a/rq/executions.py
+++ b/rq/executions.py
@@ -9,7 +9,7 @@
 
 from .job import Job
 from .registry import BaseRegistry, StartedJobRegistry
-from .utils import as_text, current_timestamp, now
+from .utils import as_text, current_timestamp, now, parse_composite_key
 
 
 # TODO: add execution.worker
@@ -63,8 +63,8 @@ def refresh(self):
     @classmethod
     def from_composite_key(cls, composite_key: str, connection: Redis) -> 'Execution':
         """A combination of job_id and execution_id separated by a colon."""
-        job_id, id = composite_key.split(':')
-        return cls(id=id, job_id=job_id, connection=connection)
+        job_id, execution_id = parse_composite_key(composite_key)
+        return cls(id=execution_id, job_id=job_id, connection=connection)
 
     @classmethod
     def create(cls, job: Job, ttl: int, pipeline: 'Pipeline') -> 'Execution':
@@ -86,7 +86,7 @@ def save(self, ttl: int, pipeline: Optional['Pipeline'] = None):
     def delete(self, job: Job, pipeline: 'Pipeline'):
         """Delete an execution from Redis."""
         pipeline.delete(self.key)
-        job.started_job_registry.remove_execution(execution=self, job=job, pipeline=pipeline)
+        job.started_job_registry.remove_execution(execution=self, pipeline=pipeline)
         ExecutionRegistry(job_id=self.job_id, connection=self.connection).remove(execution=self, pipeline=pipeline)
 
     def serialize(self) -> Dict:
@@ -118,7 +118,7 @@ def __init__(self, job_id: str, connection: Redis):
         self.job_id = job_id
         self.key = self.key_template.format(job_id)
 
-    def cleanup(self, timestamp: Optional[float] = None):
+    def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optional[List] = None):
         """Remove expired jobs from registry.
 
         Removes jobs with an expiry time earlier than timestamp, specified as
diff --git a/rq/job.py b/rq/job.py
index c43340bf8..d82e6f3b4 100644
--- a/rq/job.py
+++ b/rq/job.py
@@ -1250,12 +1250,10 @@ def _remove_from_registries(self, pipeline: Optional['Pipeline'] = None, remove_
             from .registry import StartedJobRegistry
 
             # TODO: need to cleanup job executions too
-
             registry = StartedJobRegistry(
                 self.origin, connection=self.connection, job_class=self.__class__, serializer=self.serializer
             )
             registry.remove_executions(self, pipeline=pipeline)
-            registry.remove(self, pipeline=pipeline)
 
         elif self.is_scheduled:
             from .registry import ScheduledJobRegistry
@@ -1621,6 +1619,7 @@ def dependencies_are_met(
         parent_job: Optional['Job'] = None,
         pipeline: Optional['Pipeline'] = None,
         exclude_job_id: Optional[str] = None,
+        refresh_job_status: bool = True,
     ) -> bool:
         """Returns a boolean indicating if all of this job's dependencies are `FINISHED`
 
@@ -1634,7 +1633,8 @@ def dependencies_are_met(
         Args:
             parent_job (Optional[Job], optional): The parent Job. Defaults to None.
             pipeline (Optional[Pipeline], optional): The Redis' pipeline. Defaults to None.
-            exclude_job_id (Optional[str], optional): Whether to exclude the job id.. Defaults to None.
+            exclude_job_id (Optional[str], optional): Whether to exclude the job id. Defaults to None.
+            refresh_job_status (bool): whether to refresh job status when checking for dependencies. Defaults to True.
 
         Returns:
             are_met (bool): Whether the dependencies were met.
@@ -1656,9 +1656,9 @@ def dependencies_are_met(
             # If parent job is not finished, we should only continue
             # if this job allows parent job to fail
             dependencies_ids.discard(parent_job.id)
-            if parent_job.get_status() == JobStatus.CANCELED:
+            if parent_job.get_status(refresh=refresh_job_status) == JobStatus.CANCELED:
                 return False
-            elif parent_job._status == JobStatus.FAILED and not self.allow_dependency_failures:
+            elif parent_job.get_status(refresh=False) == JobStatus.FAILED and not self.allow_dependency_failures:
                 return False
 
             # If the only dependency is parent job, dependency has been met
diff --git a/rq/queue.py b/rq/queue.py
index a34a1557a..edcb9c045 100644
--- a/rq/queue.py
+++ b/rq/queue.py
@@ -1192,7 +1192,11 @@ def run_sync(self, job: 'Job') -> 'Job':
         return job
 
     def enqueue_dependents(
-        self, job: 'Job', pipeline: Optional['Pipeline'] = None, exclude_job_id: Optional[str] = None
+        self,
+        job: 'Job',
+        pipeline: Optional['Pipeline'] = None,
+        exclude_job_id: Optional[str] = None,
+        refresh_job_status: bool = True,
     ):
         """Enqueues all jobs in the given job's dependents set and clears it.
 
@@ -1204,6 +1208,7 @@ def enqueue_dependents(
             job (Job): The Job to enqueue the dependents
             pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             exclude_job_id (Optional[str], optional): Whether to exclude the job id. Defaults to None.
+            refresh_job_status (bool): whether to refresh job status when checking for dependencies. Defaults to True.
         """
         from .registry import DeferredJobRegistry
 
@@ -1233,6 +1238,7 @@ def enqueue_dependents(
                         parent_job=job,
                         pipeline=pipe,
                         exclude_job_id=exclude_job_id,
+                        refresh_job_status=refresh_job_status,
                     )
                     and dependent_job.get_status(refresh=False) != JobStatus.CANCELED
                 ]
@@ -1315,7 +1321,7 @@ def lpop(cls, queue_keys: Sequence[str], timeout: Optional[int], connection: Opt
             if timeout == 0:
                 raise ValueError('RQ does not support indefinite timeouts. Please pick a timeout value > 0')
             colored_queues = ', '.join(map(str, [green(str(queue)) for queue in queue_keys]))
-            logger.debug("Starting BLPOP operation for queues %s with timeout of %d", colored_queues, timeout)
+            logger.debug('Starting BLPOP operation for queues %s with timeout of %d', colored_queues, timeout)
             assert connection
             result = connection.blpop(queue_keys, timeout)
             if result is None:
diff --git a/rq/registry.py b/rq/registry.py
index d451d5b09..bfd6ba279 100644
--- a/rq/registry.py
+++ b/rq/registry.py
@@ -2,8 +2,9 @@
 import logging
 import time
 import traceback
+import warnings
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, Type, Union, cast
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
 
 from rq.serializers import resolve_serializer
 
@@ -12,7 +13,7 @@
 from .job import Job, JobStatus
 from .queue import Queue
 from .timeouts import BaseDeathPenalty, UnixSignalDeathPenalty
-from .utils import as_text, backend_class, current_timestamp
+from .utils import as_text, backend_class, current_timestamp, now, parse_composite_key
 
 if TYPE_CHECKING:
     from redis import Redis
@@ -34,7 +35,6 @@ class BaseRegistry:
     job_class = Job
     death_penalty_class = UnixSignalDeathPenalty
     key_template = 'rq:registry:{0}'
-    cleanup: Callable[..., Any]
 
     def __init__(
         self,
@@ -103,19 +103,19 @@ def get_job_count(self, cleanup=True) -> int:
             self.cleanup()
         return self.connection.zcard(self.key)
 
-    def add(self, job: 'Job', ttl=0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+    def add(self, job: Job, ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
         """Adds a job to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            job (Job): The Job to add
+            job (Job): The Job to add, or job_id.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
         if pipeline is not None:
@@ -123,12 +123,12 @@ def add(self, job: 'Job', ttl=0, pipeline: Optional['Pipeline'] = None, xx: bool
 
         return self.connection.zadd(self.key, {job.id: score}, xx=xx)
 
-    def remove(self, job: Union['Job', str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            job (Job|str): The Job to remove from the registry, or job_id
+            pipeline (Pipeline|None): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
@@ -142,18 +142,6 @@ def remove(self, job: Union['Job', str], pipeline: Optional['Pipeline'] = None,
             job_instance.delete()
         return result
 
-    def remove_executions(self, job: 'Job', pipeline: Optional['Pipeline'] = None):
-        """Removes job executions from registry
-
-        Args:
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
-        """
-        connection = pipeline if pipeline is not None else self.connection
-        execution_ids = [execution.composite_key for execution in job.get_executions()]
-        if execution_ids:
-            return connection.zrem(self.key, *execution_ids)
-
     def get_expired_job_ids(self, timestamp: Optional[float] = None):
         """Returns job ids whose score are less than current timestamp.
 
@@ -163,23 +151,23 @@ def get_expired_job_ids(self, timestamp: Optional[float] = None):
         """
         score = timestamp if timestamp is not None else current_timestamp()
         expired_jobs = self.connection.zrangebyscore(self.key, 0, score)
-        return [as_text(job_id) for job_id in expired_jobs]
+        return [self.parse_job_id(job_id) for job_id in expired_jobs]
 
     def get_job_ids(self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True) -> List[str]:
         """Returns list of all job ids.
 
         Args:
-            start (int, optional): _description_. Defaults to 0.
-            end (int, optional): _description_. Defaults to -1.
-            desc (bool, optional): _description_. Defaults to False.
-            cleanup (bool, optional): _description_. Defaults to True.
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
 
         Returns:
-            _type_: _description_
+            List[str]: list of the job ids in the registry
         """
         if cleanup:
             self.cleanup()
-        return [as_text(job_id) for job_id in self.connection.zrange(self.key, start, end, desc=desc)]
+        return [self.parse_job_id(job_id) for job_id in self.connection.zrange(self.key, start, end, desc=desc)]
 
     def get_queue(self):
         """Returns Queue object associated with this registry."""
@@ -228,6 +216,26 @@ def requeue(self, job_or_id: Union['Job', str], at_front: bool = False) -> 'Job'
             pipeline.execute()
         return job
 
+    @staticmethod
+    def parse_job_id(entry: str) -> str:
+        """Generic function to retrieve the job id from the stored entry.
+        Some Registries might have a different entry format.
+
+        Args:
+            entry (str): the entry from the registry
+
+        Returns:
+            str: the job_id parsed from the registry.
+        """
+        # base registry only stores job_ids as is.
+        if not isinstance(entry, str):
+            entry = as_text(entry)  # type: ignore
+        return entry
+
+    def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optional[List] = None):
+        """This method is automatically called by `count()` and `get_job_ids()` methods
+        implemented in BaseRegistry. Base registry doesn't have any special cleanup instructions"""
+
 
 class StartedJobRegistry(BaseRegistry):
     """
@@ -237,6 +245,8 @@ class StartedJobRegistry(BaseRegistry):
 
     Jobs are added to registry right before they are executed and removed
     right after completion (success or failure).
+
+    Each entry is a {job_id}:{execution_id}
     """
 
     key_template = 'rq:wip:{0}'
@@ -255,7 +265,6 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
         job_ids = self.get_expired_job_ids(score)
 
         if job_ids:
-            failed_job_registry = FailedJobRegistry(self.name, self.connection, serializer=self.serializer)
             queue = self.get_queue()
 
             with self.connection.pipeline() as pipeline:
@@ -286,59 +295,119 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
-        return job_ids
-
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Execution',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ) -> None:
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Execution): The Execution to remove
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        connection.zrem(self.key, execution.composite_key)
         # if delete_job:
         #     job.delete()
-        return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Function to retrieve a list of tuples where the first item is the job id and
+            the second is the execution id.
+
+        Args:
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
+
+        Returns:
+            List[Tuple[str, str]]: a list of tuples where the first item is the job id and
+            the second is the execution id.
+        """
+        if cleanup:
+            self.cleanup()
+        return [
+            parse_composite_key(as_text(entry)) for entry in self.connection.zrange(self.key, start, end, desc=desc)
+        ]
+
+    def __contains__(self, item: Any) -> bool:
+        """Method to check if the item is in the registry.
+
+        Args:
+            item (Any): Either a Job (instance of job_class) or a job id.
+
+        Returns:
+            bool: True if the item is in the registry.
+        """
+        job_id = item
+        if isinstance(item, self.job_class):
+            job_id = item.id
+        return cast(str, job_id) in self.get_job_ids(cleanup=False)
+
+    def add(self, job: Job, ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+        raise NotImplementedError()
+
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
+        raise NotImplementedError()
+
+    @staticmethod
+    def parse_job_id(entry: str) -> str:
+        # other classes might have a different entry format
+        # base registry stores just the job id
+        if not isinstance(entry, str):
+            entry = as_text(entry)  # type: ignore
+        job_id, _execution_id = parse_composite_key(entry)
+        return job_id
+
+    def remove_executions(self, job: Job, pipeline: Optional['Pipeline'] = None) -> None:
+        """Removes job executions from the started job registry.
+
+        Args:
+            job (Job): The Job to remove from the registry
+            pipeline (Optional['Pipeline']): The Redis Pipeline. Defaults to None.
+        """
+        connection = pipeline if pipeline is not None else self.connection
+        execution_ids = [execution.composite_key for execution in job.get_executions()]
+        if execution_ids:
+            connection.zrem(self.key, *execution_ids)
 
 
 class FinishedJobRegistry(BaseRegistry):
@@ -349,7 +418,7 @@ class FinishedJobRegistry(BaseRegistry):
 
     key_template = 'rq:finished:{0}'
 
-    def cleanup(self, timestamp: Optional[float] = None):
+    def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optional[List] = None):
         """Remove expired jobs from registry.
 
         Removes jobs with an expiry time earlier than timestamp, specified as
@@ -367,7 +436,7 @@ class FailedJobRegistry(BaseRegistry):
 
     key_template = 'rq:failed:{0}'
 
-    def cleanup(self, timestamp: Optional[float] = None):
+    def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optional[List] = None):
         """Remove expired jobs from registry.
 
         Removes jobs with an expiry time earlier than timestamp, specified as
@@ -414,7 +483,7 @@ class DeferredJobRegistry(BaseRegistry):
 
     key_template = 'rq:deferred:{0}'
 
-    def cleanup(self, timestamp=None):
+    def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optional[List] = None):
         """Remove expired jobs from registry and add them to FailedJobRegistry.
         Removes jobs with an expiry time earlier than timestamp, specified as
         seconds since the Unix epoch. timestamp defaults to call time if
@@ -424,8 +493,6 @@ def cleanup(self, timestamp=None):
         job_ids = self.get_expired_job_ids(score)
 
         if job_ids:
-            failed_job_registry = FailedJobRegistry(self.name, self.connection, serializer=self.serializer)
-
             with self.connection.pipeline() as pipeline:
                 for job_id in job_ids:
                     try:
@@ -434,14 +501,12 @@ def cleanup(self, timestamp=None):
                         continue
 
                     job.set_status(JobStatus.FAILED, pipeline=pipeline)
-                    exc_info = 'Expired in DeferredJobRegistry, moved to FailedJobRegistry at %s' % datetime.now()
-                    failed_job_registry.add(job, job.failure_ttl, exc_info, pipeline, True)
+                    exc_info = f'Expired in DeferredJobRegistry, moved to FailedJobRegistry at {now}'
+                    job._handle_failure(exc_string=exc_info, pipeline=pipeline)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
-        return job_ids
-
     def add(self, job, ttl=None, pipeline=None, xx=False):
         """
         Adds a job to a registry with expiry time of now + ttl.
@@ -470,7 +535,7 @@ def __init__(self, *args, **kwargs):
     def schedule(self, job: 'Job', scheduled_datetime, pipeline: Optional['Pipeline'] = None):
         """
         Adds job to registry, scored by its execution time (in UTC).
-        If datetime has no tzinfo, it will assume localtimezone.
+        If datetime has no tzinfo, it will assume local timezone.
         """
         # If datetime has no timezone, assume server's local timezone
         if not scheduled_datetime.tzinfo:
@@ -480,21 +545,18 @@ def schedule(self, job: 'Job', scheduled_datetime, pipeline: Optional['Pipeline'
         timestamp = calendar.timegm(scheduled_datetime.utctimetuple())
         return self.connection.zadd(self.key, {job.id: timestamp})
 
-    def cleanup(self):
-        """This method is only here to prevent errors because this method is
-        automatically called by `count()` and `get_job_ids()` methods
-        implemented in BaseRegistry."""
-        pass
-
-    def remove_jobs(self, timestamp: Optional[datetime] = None, pipeline: Optional['Pipeline'] = None):
+    def remove_jobs(self, timestamp: Optional[int] = None, pipeline: Optional['Pipeline'] = None):
         """Remove jobs whose timestamp is in the past from registry.
 
         Args:
-            timestamp (Optional[datetime], optional): The timestamp. Defaults to None.
-            pipeline (Optional[Pipeline], optional): The Redis pipeline. Defaults to None.
+            timestamp (Optional[int], optional): The timestamp. Defaults to None.
+            pipeline (Optional['Pipeline'], optional): The Redis pipeline. Defaults to None.
         """
+        warnings.warn(
+            'ScheduledJobRegistry.remove_jobs() is deprecated and will be removed in the future.', DeprecationWarning
+        )
         connection = pipeline if pipeline is not None else self.connection
-        score: Any = timestamp if timestamp is not None else current_timestamp()
+        score: int = timestamp if timestamp is not None else current_timestamp()
         return connection.zremrangebyscore(self.key, 0, score)
 
     def get_jobs_to_schedule(self, timestamp: Optional[int] = None, chunk_size: int = 1000) -> List[str]:
@@ -541,12 +603,6 @@ class CanceledJobRegistry(BaseRegistry):
     def get_expired_job_ids(self, timestamp: Optional[float] = None):
         raise NotImplementedError
 
-    def cleanup(self):
-        """This method is only here to prevent errors because this method is
-        automatically called by `count()` and `get_job_ids()` methods
-        implemented in BaseRegistry."""
-        pass
-
 
 def clean_registries(queue: 'Queue', exception_handlers: Optional[list] = None):
     """Cleans StartedJobRegistry, FinishedJobRegistry and FailedJobRegistry, and DeferredJobRegistry of a queue.
diff --git a/rq/utils.py b/rq/utils.py
index 8491e8d67..cdd6cf772 100644
--- a/rq/utils.py
+++ b/rq/utils.py
@@ -4,13 +4,12 @@
 The formatter for ANSI colored console output is heavily based on Pygments
 terminal colorizing code, originally by Georg Brandl.
 """
-
 import calendar
 import datetime
-import datetime as dt
 import importlib
 import logging
 import numbers
+import warnings
 
 # TODO: Change import path to "collections.abc" after we stop supporting Python 3.8
 from typing import (
@@ -153,11 +152,11 @@ def now() -> datetime.datetime:
 _TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
 
 
-def utcformat(dt: dt.datetime) -> str:
+def utcformat(dt: datetime.datetime) -> str:
     return dt.strftime(as_text(_TIMESTAMP_FORMAT))
 
 
-def utcparse(string: str) -> dt.datetime:
+def utcparse(string: str) -> datetime.datetime:
     try:
         parsed = datetime.datetime.strptime(string, _TIMESTAMP_FORMAT)
     except ValueError:
@@ -225,7 +224,7 @@ def backend_class(holder, default_name, override=None) -> Type:
         return override
 
 
-def str_to_date(date_str: Optional[bytes]) -> Optional[dt.datetime]:
+def str_to_date(date_str: Optional[bytes]) -> Optional[datetime.datetime]:
     if not date_str:
         return None
     else:
@@ -375,3 +374,26 @@ def get_connection_from_queues(queues_or_names: Iterable[Union[str, 'Queue']]) -
         if isinstance(queue_or_name, Queue):
             return queue_or_name.connection
     return None
+
+
+def parse_composite_key(composite_key: str) -> Tuple[str, str]:
+    """Method returns a parsed composite key.
+
+    Args:
+        composite_key (str): the composite key to parse
+
+    Returns:
+        tuple[str, str]: tuple of job id and the execution id
+    """
+    result = composite_key.split(':')
+    if len(result) == 1:
+        # StartedJobRegistry contains a composite key under the sorted set
+        # a single job_id should've never ended up in the set, but
+        # just in case there's a regression (tests don't show any)
+        warnings.warn(
+            f'Composite key must contain job_id:execution_id, got {composite_key}',
+            DeprecationWarning,
+        )
+        return (result[0], '')
+    job_id, execution_id = result
+    return (job_id, execution_id)
diff --git a/rq/worker.py b/rq/worker.py
index 3c16d73c7..ed71f0b05 100644
--- a/rq/worker.py
+++ b/rq/worker.py
@@ -657,16 +657,11 @@ def work(
 
     def cleanup_execution(self, job: 'Job', pipeline: 'Pipeline'):
         """Cleans up the execution of a job.
-        It will remove the job from the `StartedJobRegistry` and delete the Execution object.
+        It will remove the job execution record from the `StartedJobRegistry` and delete the Execution object.
         """
         self.log.debug('Cleaning up execution of job %s', job.id)
-        started_job_registry = StartedJobRegistry(
-            job.origin, self.connection, job_class=self.job_class, serializer=self.serializer
-        )
         self.set_current_job_id(None, pipeline=pipeline)
-        started_job_registry.remove(job, pipeline=pipeline)
-        if self.execution:
-            started_job_registry.remove_execution(self.execution, job=job, pipeline=pipeline)
+        if self.execution is not None:
             self.execution.delete(job=job, pipeline=pipeline)
             self.execution = None
 
diff --git a/tests/test_executions.py b/tests/test_executions.py
index 8103ab639..59f715823 100644
--- a/tests/test_executions.py
+++ b/tests/test_executions.py
@@ -1,6 +1,8 @@
 from time import sleep
+from unittest.mock import patch
 
 from rq.executions import Execution, ExecutionRegistry
+from rq.job import Job
 from rq.queue import Queue
 from rq.utils import current_timestamp, now
 from rq.worker import Worker
@@ -9,13 +11,15 @@
 
 
 class TestRegistry(RQTestCase):
+    """Test the execution registry."""
+
     def setUp(self):
         super().setUp()
+        self.queue = Queue(connection=self.connection)
 
     def test_equality(self):
         """Test equality between Execution objects"""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello)
+        job = self.queue.enqueue(say_hello)
         pipeline = self.connection.pipeline()
         execution_1 = Execution.create(job=job, ttl=100, pipeline=pipeline)
         execution_2 = Execution.create(job=job, ttl=100, pipeline=pipeline)
@@ -26,8 +30,7 @@ def test_equality(self):
 
     def test_add_delete_executions(self):
         """Test adding and deleting executions"""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello)
+        job = self.queue.enqueue(say_hello)
         pipeline = self.connection.pipeline()
         execution = Execution.create(job=job, ttl=100, pipeline=pipeline)
         pipeline.execute()
@@ -48,8 +51,7 @@ def test_add_delete_executions(self):
 
     def test_execution_registry(self):
         """Test the ExecutionRegistry class"""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello)
+        job = self.queue.enqueue(say_hello)
         registry = ExecutionRegistry(job_id=job.id, connection=self.connection)
 
         pipeline = self.connection.pipeline()
@@ -66,25 +68,23 @@ def test_execution_registry(self):
 
     def test_ttl(self):
         """Execution registry and job execution should follow heartbeat TTL"""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello, timeout=-1)
-        worker = Worker([queue], connection=self.connection)
+        job = self.queue.enqueue(say_hello, timeout=-1)
+        worker = Worker([self.queue], connection=self.connection)
         execution = worker.prepare_execution(job=job)
         self.assertTrue(self.connection.ttl(job.execution_registry.key) >= worker.get_heartbeat_ttl(job))
         self.assertTrue(self.connection.ttl(execution.key) >= worker.get_heartbeat_ttl(job))
 
     def test_heartbeat(self):
         """Test heartbeat should refresh execution as well as registry TTL"""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello, timeout=1)
-        worker = Worker([queue], connection=self.connection)
+        job = self.queue.enqueue(say_hello, timeout=1)
+        worker = Worker([self.queue], connection=self.connection)
         execution = worker.prepare_execution(job=job)
 
         # The actual TTL should be 150 seconds
         self.assertTrue(1 < self.connection.ttl(job.execution_registry.key) < 160)
         self.assertTrue(1 < self.connection.ttl(execution.key) < 160)
         with self.connection.pipeline() as pipeline:
-            worker.execution.heartbeat(job.started_job_registry, 200, pipeline)  # type: ignore
+            worker.execution.heartbeat(job.started_job_registry, 200, pipeline)
             pipeline.execute()
 
         # The actual TTL should be 260 seconds for registry and 200 seconds for execution
@@ -93,9 +93,8 @@ def test_heartbeat(self):
 
     def test_registry_cleanup(self):
         """ExecutionRegistry.cleanup() should remove expired executions."""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello)
-        worker = Worker([queue], connection=self.connection)
+        job = self.queue.enqueue(say_hello)
+        worker = Worker([self.queue], connection=self.connection)
         worker.prepare_execution(job=job)
 
         registry = job.execution_registry
@@ -113,26 +112,24 @@ def test_registry_cleanup(self):
 
     def test_delete_registry(self):
         """ExecutionRegistry.delete() should delete registry and its executions."""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello)
-        worker = Worker([queue], connection=self.connection)
+        job = self.queue.enqueue(say_hello)
+        worker = Worker([self.queue], connection=self.connection)
         execution = worker.prepare_execution(job=job)
 
-        self.assertIn(execution.composite_key, job.started_job_registry.get_job_ids())
+        self.assertIn(execution.job_id, job.started_job_registry.get_job_ids())
 
         registry = job.execution_registry
         pipeline = self.connection.pipeline()
         registry.delete(job=job, pipeline=pipeline)
         pipeline.execute()
 
-        self.assertNotIn(execution.composite_key, job.started_job_registry.get_job_ids())
+        self.assertNotIn(execution.job_id, job.started_job_registry.get_job_ids())
         self.assertFalse(self.connection.exists(registry.key))
 
     def test_get_execution_ids(self):
         """ExecutionRegistry.get_execution_ids() should return a list of execution IDs"""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello)
-        worker = Worker([queue], connection=self.connection)
+        job = self.queue.enqueue(say_hello)
+        worker = Worker([self.queue], connection=self.connection)
 
         execution = worker.prepare_execution(job=job)
         execution_2 = worker.prepare_execution(job=job)
@@ -142,20 +139,19 @@ def test_get_execution_ids(self):
 
     def test_execution_added_to_started_job_registry(self):
         """Ensure worker adds execution to started job registry"""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(long_running_job, timeout=3)
-        Worker([queue], connection=self.connection)
+        job = self.queue.enqueue(long_running_job, timeout=3)
+        Worker([self.queue], connection=self.connection)
 
         # Start worker process in background with 1 second monitoring interval
         process = start_worker_process(
-            queue.name, worker_name='w1', connection=self.connection, burst=True, job_monitoring_interval=1
+            self.queue.name, worker_name='w1', connection=self.connection, burst=True, job_monitoring_interval=1
         )
 
         sleep(0.5)
         # Execution should be registered in started job registry
         execution = job.get_executions()[0]
         self.assertEqual(len(job.get_executions()), 1)
-        self.assertIn(execution.composite_key, job.started_job_registry.get_job_ids())
+        self.assertIn(execution.job_id, job.started_job_registry.get_job_ids())
 
         last_heartbeat = execution.last_heartbeat
         last_heartbeat = now()
@@ -173,9 +169,8 @@ def test_execution_added_to_started_job_registry(self):
 
     def test_fetch_execution(self):
         """Ensure Execution.fetch() fetches the correct execution"""
-        queue = Queue(connection=self.connection)
-        job = queue.enqueue(say_hello)
-        worker = Worker([queue], connection=self.connection)
+        job = self.queue.enqueue(say_hello)
+        worker = Worker([self.queue], connection=self.connection)
         execution = worker.prepare_execution(job=job)
 
         fetched_execution = Execution.fetch(id=execution.id, job_id=job.id, connection=self.connection)
@@ -185,3 +180,30 @@ def test_fetch_execution(self):
         # Execution.fetch raises ValueError if execution is not found
         with self.assertRaises(ValueError):
             Execution.fetch(id=execution.id, job_id=job.id, connection=self.connection)
+
+    def test_init_from_composite_key(self):
+        """Ensure the from_composite_key can correctly parse job_id and execution_id"""
+        composite_key = 'job_id:execution_id'
+        execution = Execution.from_composite_key(composite_key, connection=self.connection)
+        self.assertEqual(execution.job_id, 'job_id')
+        self.assertEqual(execution.id, 'execution_id')
+
+    def test_job_auto_fetch(self):
+        """Ensure that if the job is not set, the Job.fetch is not called"""
+        job = self.queue.enqueue(say_hello)
+        execution = Execution('execution_id', job.id, connection=self.connection)
+
+        with patch.object(Job, 'fetch') as mock:
+            mock.return_value = Job(id=job.id, connection=self.connection)
+            # the first call would fetch the job
+            first_fetch = execution.job
+            self.assertEqual(first_fetch.id, job.id)
+            self.assertEqual(mock.call_count, 1)
+            self.assertNotEqual(id(job), id(first_fetch))
+
+            # the second call should return the same object
+            second_fetch = execution.job
+            self.assertEqual(second_fetch.id, job.id)
+            # call count remains the same
+            self.assertEqual(mock.call_count, 1)
+            self.assertEqual(id(first_fetch), id(second_fetch))
diff --git a/tests/test_intermediate_queue.py b/tests/test_intermediate_queue.py
index 1c8eb6b7a..dad8b4406 100644
--- a/tests/test_intermediate_queue.py
+++ b/tests/test_intermediate_queue.py
@@ -2,10 +2,12 @@
 from datetime import datetime, timedelta, timezone
 from unittest.mock import patch
 
+import pytest
 from redis import Redis
 
 from rq import Queue, Worker
 from rq.intermediate_queue import IntermediateQueue
+from rq.job import JobStatus
 from rq.maintenance import clean_intermediate_queue
 from rq.utils import get_version
 from tests import RQTestCase
@@ -14,11 +16,15 @@
 
 @unittest.skipIf(get_version(Redis()) < (6, 2, 0), 'Skip if Redis server < 6.2.0')
 class TestIntermediateQueue(RQTestCase):
+    def setUp(self):
+        super().setUp()
+        self.queue = Queue('foo', connection=self.connection)
+        self.intermediate_queue = IntermediateQueue(self.queue.key, connection=self.connection)
+
     def test_set_first_seen(self):
         """Ensure that the first_seen attribute is set correctly."""
-        queue = Queue('foo', connection=self.connection)
-        intermediate_queue = IntermediateQueue(queue.key, connection=self.connection)
-        job = queue.enqueue(say_hello)
+        intermediate_queue = self.intermediate_queue
+        job = self.queue.enqueue(say_hello)
 
         # set_first_seen() should only succeed the first time around
         self.assertTrue(intermediate_queue.set_first_seen(job.id))
@@ -29,22 +35,21 @@ def test_set_first_seen(self):
 
     def test_get_first_seen(self):
         """Ensure that the first_seen attribute is set correctly."""
-        queue = Queue('foo', connection=self.connection)
-        intermediate_queue = IntermediateQueue(queue.key, connection=self.connection)
-        job = queue.enqueue(say_hello)
+        intermediate_queue = self.intermediate_queue
+        job = self.queue.enqueue(say_hello)
 
         self.assertIsNone(intermediate_queue.get_first_seen(job.id))
 
         # Check first seen was set correctly
         intermediate_queue.set_first_seen(job.id)
         timestamp = intermediate_queue.get_first_seen(job.id)
-        self.assertTrue(datetime.now(tz=timezone.utc) - timestamp < timedelta(seconds=5))  # type: ignore
+        assert timestamp
+        self.assertTrue(datetime.now(tz=timezone.utc) - timestamp < timedelta(seconds=5))
 
     def test_should_be_cleaned_up(self):
         """Job in the intermediate queue should be cleaned up if it was seen more than 1 minute ago."""
-        queue = Queue('foo', connection=self.connection)
-        intermediate_queue = IntermediateQueue(queue.key, connection=self.connection)
-        job = queue.enqueue(say_hello)
+        intermediate_queue = self.intermediate_queue
+        job = self.queue.enqueue(say_hello)
 
         # Returns False if there's no first seen timestamp
         self.assertFalse(intermediate_queue.should_be_cleaned_up(job.id))
@@ -59,23 +64,25 @@ def test_should_be_cleaned_up(self):
 
     def test_get_job_ids(self):
         """Dequeueing job from a single queue moves job to intermediate queue."""
-        queue = Queue('foo', connection=self.connection)
-        job_1 = queue.enqueue(say_hello)
-
-        intermediate_queue = IntermediateQueue(queue.key, connection=self.connection)
+        intermediate_queue = self.intermediate_queue
+        job_1 = self.queue.enqueue(say_hello)
 
         # Ensure that the intermediate queue is empty
         self.connection.delete(intermediate_queue.key)
 
         # Job ID is not in intermediate queue
         self.assertEqual(intermediate_queue.get_job_ids(), [])
-        job, queue = Queue.dequeue_any([queue], timeout=None, connection=self.connection)
+        result = Queue.dequeue_any([self.queue], timeout=None, connection=self.connection)
+        assert result
+        _job, queue = result
         # After job is dequeued, the job ID is in the intermediate queue
         self.assertEqual(intermediate_queue.get_job_ids(), [job_1.id])
 
         # Test the blocking version
         job_2 = queue.enqueue(say_hello)
-        job, queue = Queue.dequeue_any([queue], timeout=1, connection=self.connection)
+        result = Queue.dequeue_any([queue], timeout=1, connection=self.connection)
+        assert result
+        _job, queue = result
         # After job is dequeued, the job ID is in the intermediate queue
         self.assertEqual(intermediate_queue.get_job_ids(), [job_1.id, job_2.id])
 
@@ -85,27 +92,25 @@ def test_get_job_ids(self):
 
     def test_cleanup_intermediate_queue_in_maintenance(self):
         """Ensure jobs stuck in the intermediate queue are cleaned up."""
-        queue = Queue('foo', connection=self.connection)
-        job = queue.enqueue(say_hello)
-
-        intermediate_queue = IntermediateQueue(queue.key, connection=self.connection)
+        intermediate_queue = self.intermediate_queue
+        job = self.queue.enqueue(say_hello)
         self.connection.delete(intermediate_queue.key)
 
         # If job execution fails after it's dequeued, job should be in the intermediate queue
         # and it's status is still QUEUED
         with patch.object(Worker, 'execute_job'):
-            worker = Worker(queue, connection=self.connection)
+            worker = Worker(self.queue, connection=self.connection)
             worker.work(burst=True)
 
             # If worker.execute_job() does nothing, job status should be `queued`
             # even though it's not in the queue, but it should be in the intermediate queue
-            self.assertEqual(job.get_status(), 'queued')
-            self.assertFalse(job.id in queue.get_job_ids())
+            self.assertEqual(job.get_status(), JobStatus.QUEUED)
+            self.assertFalse(job.id in self.queue.get_job_ids())
             self.assertEqual(intermediate_queue.get_job_ids(), [job.id])
 
             self.assertIsNone(intermediate_queue.get_first_seen(job.id))
-            clean_intermediate_queue(worker, queue)
-            # After clean_intermediate_queue is called, the job should be marked as seen,
+            intermediate_queue.cleanup(worker, self.queue)
+            # After intermediate_queue.cleanup is called, the job should be marked as seen,
             # but since it's been less than 1 minute, it should not be cleaned up
             self.assertIsNotNone(intermediate_queue.get_first_seen(job.id))
             self.assertFalse(intermediate_queue.should_be_cleaned_up(job.id))
@@ -116,42 +121,41 @@ def test_cleanup_intermediate_queue_in_maintenance(self):
             two_minutes_ago = datetime.now(tz=timezone.utc) - timedelta(minutes=2)
             self.connection.set(first_seen_key, two_minutes_ago.timestamp(), ex=10)
 
-            clean_intermediate_queue(worker, queue)
+            intermediate_queue.cleanup(worker, self.queue)
             self.assertEqual(intermediate_queue.get_job_ids(), [])
             self.assertEqual(job.get_status(), 'failed')
 
-            job = queue.enqueue(say_hello)
+            job = self.queue.enqueue(say_hello)
             worker.work(burst=True)
             self.assertEqual(intermediate_queue.get_job_ids(), [job.id])
 
             # If job is gone, it should be immediately removed from the intermediate queue
             job.delete()
-            clean_intermediate_queue(worker, queue)
+            intermediate_queue.cleanup(worker, self.queue)
             self.assertEqual(intermediate_queue.get_job_ids(), [])
 
     def test_cleanup_intermediate_queue(self):
         """Ensure jobs stuck in the intermediate queue are cleaned up."""
-        queue = Queue('foo', connection=self.connection)
-        job = queue.enqueue(say_hello)
+        intermediate_queue = self.intermediate_queue
+        job = self.queue.enqueue(say_hello)
 
-        intermediate_queue = IntermediateQueue(queue.key, connection=self.connection)
         self.connection.delete(intermediate_queue.key)
 
         # If job execution fails after it's dequeued, job should be in the intermediate queue
         # and it's status is still QUEUED
         with patch.object(Worker, 'execute_job'):
-            worker = Worker(queue, connection=self.connection)
+            worker = Worker(self.queue, connection=self.connection)
             worker.work(burst=True)
 
             # If worker.execute_job() does nothing, job status should be `queued`
             # even though it's not in the queue, but it should be in the intermediate queue
-            self.assertEqual(job.get_status(), 'queued')
-            self.assertFalse(job.id in queue.get_job_ids())
+            self.assertEqual(job.get_status(), JobStatus.QUEUED)
+            self.assertFalse(job.id in self.queue.get_job_ids())
             self.assertEqual(intermediate_queue.get_job_ids(), [job.id])
 
             self.assertIsNone(intermediate_queue.get_first_seen(job.id))
-            intermediate_queue.cleanup(worker, queue)
-            # After clean_intermediate_queue is called, the job should be marked as seen,
+            intermediate_queue.cleanup(worker, self.queue)
+            # After intermediate_queue.cleanup is called, the job should be marked as seen,
             # but since it's been less than 1 minute, it should not be cleaned up
             self.assertIsNotNone(intermediate_queue.get_first_seen(job.id))
             self.assertFalse(intermediate_queue.should_be_cleaned_up(job.id))
@@ -162,15 +166,53 @@ def test_cleanup_intermediate_queue(self):
             two_minutes_ago = datetime.now(tz=timezone.utc) - timedelta(minutes=2)
             self.connection.set(first_seen_key, two_minutes_ago.timestamp(), ex=10)
 
-            intermediate_queue.cleanup(worker, queue)
+            intermediate_queue.cleanup(worker, self.queue)
             self.assertEqual(intermediate_queue.get_job_ids(), [])
             self.assertEqual(job.get_status(), 'failed')
 
-            job = queue.enqueue(say_hello)
+            job = self.queue.enqueue(say_hello)
             worker.work(burst=True)
             self.assertEqual(intermediate_queue.get_job_ids(), [job.id])
 
             # If job is gone, it should be immediately removed from the intermediate queue
             job.delete()
-            intermediate_queue.cleanup(worker, queue)
+            intermediate_queue.cleanup(worker, self.queue)
             self.assertEqual(intermediate_queue.get_job_ids(), [])
+
+    def test_no_cleanup_while_in_started_queue(self):
+        """Ensure jobs stuck in the intermediate queue are cleaned up."""
+        intermediate_queue = self.intermediate_queue
+        job = self.queue.enqueue(say_hello)
+
+        self.connection.delete(intermediate_queue.key)
+
+        # If job execution fails after it's dequeued, job should be in the intermediate queue
+        # and it's status is still QUEUED
+        with patch.object(Worker, 'perform_job'):
+            worker = Worker(self.queue, connection=self.connection)
+            worker.work(burst=True)
+
+            # If worker.perform_job() does nothing, job status should be `queued`
+            # even though it's not in the queue, but it should be in the intermediate queue
+            # and the job should be in the started queue (since we only mocked perform_job)
+            self.assertEqual(job.get_status(), JobStatus.QUEUED)
+            self.assertFalse(job.id in self.queue.get_job_ids())
+            self.assertEqual(intermediate_queue.get_job_ids(), [job.id])
+
+            self.assertTrue(job.id in job.started_job_registry.get_job_ids())
+            self.assertTrue(
+                (worker.execution.job_id, worker.execution.id) in job.started_job_registry.get_job_and_execution_ids()
+            )
+
+            # this should NOT remove the job from the queue, nor set the first see key
+            # because it's still in the "execution state".
+            # perform_job was mocked and never called success or failure.
+            intermediate_queue.cleanup(worker, self.queue)
+            self.assertIsNone(intermediate_queue.get_first_seen(job.id))
+
+            self.assertTrue(job.get_status() == JobStatus.QUEUED)
+
+    def test_clean_intermediate_queue_deprecation(self):
+        with pytest.deprecated_call():
+            worker = Worker(self.queue, connection=self.connection)
+            clean_intermediate_queue(worker, self.queue)
diff --git a/tests/test_job.py b/tests/test_job.py
index 305d8b22a..40dee16f8 100644
--- a/tests/test_job.py
+++ b/tests/test_job.py
@@ -11,6 +11,7 @@
 
 from rq.defaults import CALLBACK_TIMEOUT
 from rq.exceptions import DeserializationError, InvalidJobOperation, NoSuchJobError
+from rq.executions import Execution
 from rq.job import Callback, Dependency, Job, JobStatus, cancel_job, get_current_job
 from rq.queue import Queue
 from rq.registry import (
@@ -49,7 +50,7 @@ def test_create_empty_job(self):
         # Jobs have a random UUID and a creation date
         self.assertIsNotNone(job.id)
         self.assertIsNotNone(job.created_at)
-        self.assertEqual(str(job), '<Job %s: test job>' % job.id)
+        self.assertEqual(str(job), f'<Job {job.id}: test job>')
 
         # ...and nothing else
         self.assertEqual(job.origin, '')
@@ -57,7 +58,7 @@ def test_create_empty_job(self):
         self.assertIsNone(job.started_at)
         self.assertIsNone(job.ended_at)
         self.assertIsNone(job.result)
-        self.assertIsNone(job.exc_info)
+        self.assertIsNone(job._exc_info)
 
         with self.assertRaises(DeserializationError):
             job.func
@@ -181,8 +182,7 @@ def test_fetch(self):
         self.assertIsNone(job.instance)
         self.assertEqual(job.args, (3, 4))
         self.assertEqual(job.kwargs, dict(z=2))
-        self.assertEqual(job.created_at, datetime(2012, 2, 7, 22, 13, 24, 123456,
-                         tzinfo=timezone.utc))
+        self.assertEqual(job.created_at, datetime(2012, 2, 7, 22, 13, 24, 123456, tzinfo=timezone.utc))
 
         # Job.fetch also works with execution IDs
         queue = Queue(connection=self.connection)
@@ -315,7 +315,7 @@ def test_persistence_of_callbacks(self):
     def test_store_then_fetch(self):
         """Store, then fetch."""
         job = Job.create(
-            func=fixtures.some_calculation, timeout='1h', args=(3, 4), kwargs=dict(z=2), connection=self.connection
+            func=fixtures.some_calculation, timeout=3600, args=(3, 4), kwargs=dict(z=2), connection=self.connection
         )
         job.save()
 
@@ -749,7 +749,10 @@ def test_job_delete_removes_itself_from_registries(self):
         job.save()
 
         registry = StartedJobRegistry(connection=self.connection, serializer=JSONSerializer)
-        registry.add(job, 500)
+        with self.connection.pipeline() as pipe:
+            # this will also add the execution to the registry
+            Execution.create(job, ttl=500, pipeline=pipe)
+            pipe.execute()
 
         job.delete()
         self.assertFalse(job in registry)
diff --git a/tests/test_registry.py b/tests/test_registry.py
index 78410113e..62bcf59a3 100644
--- a/tests/test_registry.py
+++ b/tests/test_registry.py
@@ -2,11 +2,15 @@
 from unittest import mock
 from unittest.mock import ANY
 
+import pytest
+
 from rq.defaults import DEFAULT_FAILURE_TTL
 from rq.exceptions import AbandonedJobError, InvalidJobOperation
+from rq.executions import Execution
 from rq.job import Dependency, Job, JobStatus, requeue_job
 from rq.queue import Queue
 from rq.registry import (
+    BaseRegistry,
     CanceledJobRegistry,
     DeferredJobRegistry,
     FailedJobRegistry,
@@ -26,57 +30,57 @@ class CustomJob(Job):
 
 
 class TestRegistry(RQTestCase):
+    """Test all the BaseRegistry functionality"""
+
     def setUp(self):
         super().setUp()
-        self.registry = StartedJobRegistry(connection=self.connection)
+        self.registry = BaseRegistry(connection=self.connection)
 
     def test_init(self):
         """Registry can be instantiated with queue or name/Redis connection"""
         queue = Queue('foo', connection=self.connection)
-        registry = StartedJobRegistry(queue=queue)
+        registry = BaseRegistry(queue=queue)
         self.assertEqual(registry.name, queue.name)
         self.assertEqual(registry.connection, queue.connection)
         self.assertEqual(registry.serializer, queue.serializer)
 
-        registry = StartedJobRegistry('bar', self.connection, serializer=JSONSerializer)
+        registry = BaseRegistry('bar', self.connection, serializer=JSONSerializer)
         self.assertEqual(registry.name, 'bar')
         self.assertEqual(registry.connection, self.connection)
         self.assertEqual(registry.serializer, JSONSerializer)
 
     def test_key(self):
-        self.assertEqual(self.registry.key, 'rq:wip:default')
+        self.assertEqual(self.registry.key, 'rq:registry:default')
 
     def test_custom_job_class(self):
-        registry = StartedJobRegistry(job_class=CustomJob)
+        registry = BaseRegistry(job_class=CustomJob)
         self.assertFalse(registry.job_class == self.registry.job_class)
 
     def test_contains(self):
-        registry = StartedJobRegistry(connection=self.connection)
         queue = Queue(connection=self.connection)
         job = queue.enqueue(say_hello)
 
-        self.assertFalse(job in registry)
-        self.assertFalse(job.id in registry)
+        self.assertFalse(job in self.registry)
+        self.assertFalse(job.id in self.registry)
 
-        registry.add(job, 5)
+        self.registry.add(job, 5)
 
-        self.assertTrue(job in registry)
-        self.assertTrue(job.id in registry)
+        self.assertTrue(job in self.registry)
+        self.assertTrue(job.id in self.registry)
 
     def test_get_expiration_time(self):
         """registry.get_expiration_time() returns correct datetime objects"""
-        registry = StartedJobRegistry(connection=self.connection)
         queue = Queue(connection=self.connection)
         job = queue.enqueue(say_hello)
 
-        registry.add(job, 5)
-        time = registry.get_expiration_time(job)
+        self.registry.add(job, 5)
+        time = self.registry.get_expiration_time(job)
         expected_time = (now() + timedelta(seconds=5)).replace(microsecond=0)
         self.assertGreaterEqual(time, expected_time - timedelta(seconds=2))
         self.assertLessEqual(time, expected_time + timedelta(seconds=2))
 
     def test_add_and_remove(self):
-        """Adding and removing job to StartedJobRegistry."""
+        """Adding and removing job from BaseRegistry."""
         timestamp = current_timestamp()
 
         queue = Queue(connection=self.connection)
@@ -118,10 +122,10 @@ def test_add_and_remove(self):
         self.assertFalse(self.connection.exists(job.key))
 
     def test_add_and_remove_with_serializer(self):
-        """Adding and removing job to StartedJobRegistry (with serializer)."""
+        """Adding and removing job from BaseRegistry (with serializer)."""
         # delete_job = True also works with job.id and custom serializer
         queue = Queue(connection=self.connection, serializer=JSONSerializer)
-        registry = StartedJobRegistry(connection=self.connection, serializer=JSONSerializer)
+        registry = BaseRegistry(connection=self.connection, serializer=JSONSerializer)
         job = queue.enqueue(say_hello)
         registry.add(job, -1)
         registry.remove(job.id, delete_job=True)
@@ -129,23 +133,15 @@ def test_add_and_remove_with_serializer(self):
         self.assertFalse(self.connection.exists(job.key))
 
     def test_get_job_ids(self):
-        """Getting job ids from StartedJobRegistry."""
+        """Getting job ids from BaseRegistry."""
         timestamp = current_timestamp()
         self.connection.zadd(self.registry.key, {'will-be-cleaned-up': 1})
         self.connection.zadd(self.registry.key, {'foo': timestamp + 10})
         self.connection.zadd(self.registry.key, {'bar': timestamp + 20})
-        self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
-
-    def test_get_job_ids_does_not_cleanup(self):
-        """Getting job ids from StartedJobRegistry without a cleanup."""
-        timestamp = current_timestamp()
-        self.connection.zadd(self.registry.key, {'will-be-returned-despite-outdated': 1})
-        self.connection.zadd(self.registry.key, {'foo': timestamp + 10})
-        self.connection.zadd(self.registry.key, {'bar': timestamp + 20})
-        self.assertEqual(self.registry.get_job_ids(cleanup=False), ['will-be-returned-despite-outdated', 'foo', 'bar'])
+        self.assertEqual(self.registry.get_job_ids(), ['will-be-cleaned-up', 'foo', 'bar'])
 
     def test_get_expired_job_ids(self):
-        """Getting expired job ids form StartedJobRegistry."""
+        """Getting expired job ids form BaseRegistry."""
         timestamp = current_timestamp()
 
         self.connection.zadd(self.registry.key, {'foo': 1})
@@ -159,106 +155,14 @@ def test_get_expired_job_ids(self):
         registry = CanceledJobRegistry(connection=self.connection)
         self.assertRaises(NotImplementedError, registry.get_expired_job_ids)
 
-    def test_cleanup_moves_jobs_to_failed_job_registry(self):
-        """Moving expired jobs to FailedJobRegistry."""
-        queue = Queue(connection=self.connection)
-        failed_job_registry = FailedJobRegistry(connection=self.connection)
-        job = queue.enqueue(say_hello)
-
-        self.connection.zadd(self.registry.key, {job.id: 2})
-
-        # Job has not been moved to FailedJobRegistry
-        self.registry.cleanup(1)
-        self.assertNotIn(job, failed_job_registry)
-        self.assertIn(job, self.registry)
-
-        with mock.patch.object(Job, 'execute_failure_callback') as mocked:
-            mock_handler = mock.MagicMock()
-            mock_handler.return_value = False
-            mock_handler_no_return = mock.MagicMock()
-            mock_handler_no_return.return_value = None
-            self.registry.cleanup(exception_handlers=[mock_handler_no_return, mock_handler])
-            mocked.assert_called_once_with(queue.death_penalty_class, AbandonedJobError, ANY, ANY)
-            mock_handler.assert_called_once_with(job, AbandonedJobError, ANY, ANY)
-            mock_handler_no_return.assert_called_once_with(job, AbandonedJobError, ANY, ANY)
-        self.assertIn(job.id, failed_job_registry)
-        self.assertNotIn(job, self.registry)
-        job.refresh()
-        self.assertEqual(job.get_status(), JobStatus.FAILED)
-        self.assertTrue(job.exc_info)  # explanation is written to exc_info
-
-    def test_job_execution(self):
-        """Job is removed from StartedJobRegistry after execution."""
-        registry = StartedJobRegistry(connection=self.connection)
-        queue = Queue(connection=self.connection)
-        worker = Worker([queue], connection=self.connection)
-
-        job = queue.enqueue(say_hello)
-        self.assertTrue(job.is_queued)
-        execution = worker.prepare_execution(job)
-        worker.prepare_job_execution(job)
-        self.assertIn(execution.composite_key, registry.get_job_ids())
-        self.assertTrue(job.is_started)
-
-        worker.perform_job(job, queue)
-        self.assertNotIn(execution.composite_key, registry.get_job_ids())
-        self.assertTrue(job.is_finished)
-
-        # Job that fails
-        job = queue.enqueue(div_by_zero)
-        execution = worker.prepare_execution(job)
-        worker.prepare_job_execution(job)
-        self.assertIn(execution.composite_key, registry.get_job_ids())
-
-        worker.perform_job(job, queue)
-        self.assertNotIn(execution.composite_key, registry.get_job_ids())
-
-    def test_remove_executions(self):
-        """Ensure all executions for a job are removed from registry."""
-        registry = StartedJobRegistry(connection=self.connection)
-        queue = Queue(connection=self.connection)
-        worker = Worker([queue], connection=self.connection)
-        job = queue.enqueue(say_hello)
-
-        execution_1 = worker.prepare_execution(job)
-        execution_2 = worker.prepare_execution(job)
-
-        self.assertIn(execution_1.composite_key, registry.get_job_ids())
-        self.assertIn(execution_2.composite_key, registry.get_job_ids())
-
-        registry.remove_executions(job)
-
-        self.assertNotIn(execution_1.composite_key, registry.get_job_ids())
-        self.assertNotIn(execution_2.composite_key, registry.get_job_ids())
-
-        job.delete()
-
-    def test_job_deletion(self):
-        """Ensure job is removed from StartedJobRegistry when deleted."""
-        registry = StartedJobRegistry(connection=self.connection)
-        queue = Queue(connection=self.connection)
-        worker = Worker([queue], connection=self.connection)
-
-        job = queue.enqueue(say_hello)
-        self.assertTrue(job.is_queued)
-
-        execution = worker.prepare_execution(job)
-        worker.prepare_job_execution(job)
-        self.assertIn(execution.composite_key, registry.get_job_ids())
-
-        pipeline = self.connection.pipeline()
-        job.delete(pipeline=pipeline)
-        pipeline.execute()
-        self.assertNotIn(execution.composite_key, registry.get_job_ids())
-
     def test_count(self):
-        """StartedJobRegistry returns the right number of job count."""
+        """BaseRegistry returns the right number of job count."""
         timestamp = current_timestamp() + 10
         self.connection.zadd(self.registry.key, {'will-be-cleaned-up': 1})
         self.connection.zadd(self.registry.key, {'foo': timestamp})
         self.connection.zadd(self.registry.key, {'bar': timestamp})
-        self.assertEqual(self.registry.count, 2)
-        self.assertEqual(len(self.registry), 2)
+        self.assertEqual(self.registry.count, 3)
+        self.assertEqual(len(self.registry), 3)
 
     def test_get_job_count(self):
         """Ensure cleanup is not called and does not affect the reported number of jobs.
@@ -283,7 +187,7 @@ def test_clean_registries(self):
         self.connection.zadd(finished_job_registry.key, {'foo': 1})
 
         started_job_registry = StartedJobRegistry(connection=self.connection)
-        self.connection.zadd(started_job_registry.key, {'foo': 1})
+        self.connection.zadd(started_job_registry.key, {'foo:execution_id': 1})
 
         failed_job_registry = FailedJobRegistry(connection=self.connection)
         self.connection.zadd(failed_job_registry.key, {'foo': 1})
@@ -302,7 +206,7 @@ def test_clean_registries_with_serializer(self):
         self.connection.zadd(finished_job_registry.key, {'foo': 1})
 
         started_job_registry = StartedJobRegistry(connection=self.connection, serializer=JSONSerializer)
-        self.connection.zadd(started_job_registry.key, {'foo': 1})
+        self.connection.zadd(started_job_registry.key, {'foo:execution_id': 1})
 
         failed_job_registry = FailedJobRegistry(connection=self.connection, serializer=JSONSerializer)
         self.connection.zadd(failed_job_registry.key, {'foo': 1})
@@ -314,53 +218,12 @@ def test_clean_registries_with_serializer(self):
 
     def test_get_queue(self):
         """registry.get_queue() returns the right Queue object."""
-        registry = StartedJobRegistry(connection=self.connection)
+        registry = BaseRegistry(connection=self.connection)
         self.assertEqual(registry.get_queue(), Queue(connection=self.connection))
 
-        registry = StartedJobRegistry('foo', connection=self.connection, serializer=JSONSerializer)
+        registry = BaseRegistry('foo', connection=self.connection, serializer=JSONSerializer)
         self.assertEqual(registry.get_queue(), Queue('foo', connection=self.connection, serializer=JSONSerializer))
 
-    def test_enqueue_dependents_when_parent_job_is_abandoned(self):
-        """Enqueuing parent job's dependencies after moving it to FailedJobRegistry due to AbandonedJobError."""
-        queue = Queue(connection=self.connection)
-        worker = Worker([queue])
-        failed_job_registry = FailedJobRegistry(connection=self.connection)
-        finished_job_registry = FinishedJobRegistry(connection=self.connection)
-        deferred_job_registry = DeferredJobRegistry(connection=self.connection)
-
-        parent_job = queue.enqueue(say_hello)
-        job_to_be_executed = queue.enqueue_call(say_hello, depends_on=Dependency(jobs=parent_job, allow_failure=True))
-        job_not_to_be_executed = queue.enqueue_call(
-            say_hello, depends_on=Dependency(jobs=parent_job, allow_failure=False)
-        )
-        self.assertIn(job_to_be_executed, deferred_job_registry)
-        self.assertIn(job_not_to_be_executed, deferred_job_registry)
-
-        self.connection.zadd(self.registry.key, {parent_job.id: 2})
-        queue.remove(parent_job.id)
-
-        with mock.patch.object(Job, 'execute_failure_callback') as mocked:
-            self.registry.cleanup()
-            mocked.assert_called_once_with(queue.death_penalty_class, AbandonedJobError, ANY, ANY)
-
-        # check that parent job was moved to FailedJobRegistry and has correct status
-        self.assertIn(parent_job, failed_job_registry)
-        self.assertNotIn(parent_job, self.registry)
-        self.assertTrue(parent_job.is_failed)
-
-        # check that only job_to_be_executed has been queued and executed
-        self.assertEqual(len(queue.get_job_ids()), 1)
-        self.assertTrue(job_to_be_executed.is_queued)
-        self.assertFalse(job_not_to_be_executed.is_queued)
-
-        worker.work(burst=True)
-        self.assertTrue(job_to_be_executed.is_finished)
-        self.assertNotIn(job_to_be_executed, deferred_job_registry)
-        self.assertIn(job_to_be_executed, finished_job_registry)
-
-        self.assertFalse(job_not_to_be_executed.is_finished)
-        self.assertNotIn(job_not_to_be_executed, finished_job_registry)
-
 
 class TestFinishedJobRegistry(RQTestCase):
     def setUp(self):
@@ -653,3 +516,211 @@ def test_worker_handle_job_failure(self):
         job = q.enqueue(div_by_zero, failure_ttl=5)
         w.handle_job_failure(job, q)
         self.assertLess(self.connection.zscore(registry.key, job.id), timestamp + 7)
+
+
+class TestStartedJobRegistry(RQTestCase):
+    def setUp(self):
+        super().setUp()
+        self.registry = StartedJobRegistry(connection=self.connection)
+        self.queue = Queue(connection=self.connection)
+
+    def test_job_deletion(self):
+        """Ensure job is removed from StartedJobRegistry when deleted."""
+        worker = Worker([self.queue], connection=self.connection)
+
+        job = self.queue.enqueue(say_hello)
+        self.assertTrue(job.is_queued)
+
+        execution = worker.prepare_execution(job)
+        worker.prepare_job_execution(job)
+        self.assertIn(execution.job_id, self.registry.get_job_ids())
+
+        pipeline = self.connection.pipeline()
+        job.delete(pipeline=pipeline)
+        pipeline.execute()
+        self.assertNotIn(execution.job_id, self.registry.get_job_ids())
+
+    def test_contains(self):
+        """Test the StartedJobRegistry __contains__ method. It is slightly different
+        because the entries in the registry are {job_id}:{execution_id} format."""
+        job = self.queue.enqueue(say_hello)
+
+        self.assertFalse(job in self.registry)
+        self.assertFalse(job.id in self.registry)
+
+        with self.connection.pipeline() as pipe:
+            self.registry.add_execution(
+                Execution(id='execution', job_id=job.id, connection=self.connection), pipeline=pipe, ttl=5
+            )
+            pipe.execute()
+        self.assertTrue(job in self.registry)
+        self.assertTrue(job.id in self.registry)
+
+    def test_infinite_score(self):
+        """Test the StartedJobRegistry __contains__ method. It is slightly different
+        because the entries in the registry are {job_id}:{execution_id} format."""
+        job = self.queue.enqueue(say_hello)
+
+        self.assertFalse(job in self.registry)
+        self.assertFalse(job.id in self.registry)
+
+        with self.connection.pipeline() as pipe:
+            execution = Execution(id='execution', job_id=job.id, connection=self.connection)
+            self.registry.add_execution(execution=execution, pipeline=pipe, ttl=-1)
+            pipe.execute()
+        self.assertTrue(job in self.registry)
+        self.assertTrue(job.id in self.registry)
+        self.assertTrue(self.connection.zscore(self.registry.key, execution.composite_key), '+inf')
+
+    def test_remove_executions(self):
+        """Ensure all executions for a job are removed from registry."""
+        worker = Worker([self.queue], connection=self.connection)
+        job = self.queue.enqueue(say_hello)
+
+        execution_1 = worker.prepare_execution(job)
+        execution_2 = worker.prepare_execution(job)
+        self.assertIn((job.id, execution_1.id), self.registry.get_job_and_execution_ids())
+        self.assertIn((job.id, execution_2.id), self.registry.get_job_and_execution_ids())
+
+        self.registry.remove_executions(job)
+
+        self.assertNotIn((job.id, execution_1.id), self.registry.get_job_and_execution_ids())
+        self.assertNotIn((job.id, execution_2.id), self.registry.get_job_and_execution_ids())
+
+        job.delete()
+
+    def test_job_execution(self):
+        """Job is removed from StartedJobRegistry after execution."""
+        worker = Worker([self.queue], connection=self.connection)
+
+        job = self.queue.enqueue(say_hello)
+        self.assertTrue(job.is_queued)
+        execution = worker.prepare_execution(job)
+        worker.prepare_job_execution(job)
+        self.assertIn(job.id, self.registry.get_job_ids())
+        self.assertIn((job.id, execution.id), self.registry.get_job_and_execution_ids())
+        self.assertTrue(job.is_started)
+
+        worker.perform_job(job, self.queue)
+        self.assertNotIn(job.id, self.registry.get_job_ids())
+        self.assertNotIn((job.id, execution.id), self.registry.get_job_and_execution_ids())
+        self.assertTrue(job.is_finished)
+
+        # Job that fails
+        job = self.queue.enqueue(div_by_zero)
+        execution = worker.prepare_execution(job)
+        worker.prepare_job_execution(job)
+        self.assertIn(job.id, self.registry.get_job_ids())
+        self.assertIn((job.id, execution.id), self.registry.get_job_and_execution_ids())
+
+        worker.perform_job(job, self.queue)
+        self.assertNotIn(job.id, self.registry.get_job_ids())
+        self.assertNotIn((job.id, execution.id), self.registry.get_job_and_execution_ids())
+
+    def test_get_job_ids(self):
+        """Getting job ids with cleanup."""
+        timestamp = current_timestamp()
+        self.connection.zadd(self.registry.key, {'will-be-cleaned-up:execution_id1': 1})
+        self.connection.zadd(self.registry.key, {'foo:execution_id2': timestamp + 10})
+        self.connection.zadd(self.registry.key, {'bar:execution_id3': timestamp + 20})
+        self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
+
+    def test_get_job_ids_does_not_cleanup(self):
+        """Getting job ids without a cleanup."""
+        timestamp = current_timestamp()
+        self.connection.zadd(self.registry.key, {'will-be-returned-despite-outdated:execution_id1': 1})
+        self.connection.zadd(self.registry.key, {'foo:execution_id2': timestamp + 10})
+        self.connection.zadd(self.registry.key, {'bar:execution_id3': timestamp + 20})
+        self.assertEqual(self.registry.get_job_ids(cleanup=False), ['will-be-returned-despite-outdated', 'foo', 'bar'])
+
+    def test_count(self):
+        """Return the right number of job count (cleanup should be performed)"""
+        timestamp = current_timestamp() + 10
+        self.connection.zadd(self.registry.key, {'will-be-cleaned-up': 1})
+        self.connection.zadd(self.registry.key, {'foo': timestamp})
+        self.connection.zadd(self.registry.key, {'bar': timestamp})
+        self.assertEqual(self.registry.count, 2)
+        self.assertEqual(len(self.registry), 2)
+
+    def test_cleanup_moves_jobs_to_failed_job_registry(self):
+        """Moving expired jobs to FailedJobRegistry."""
+
+        failed_job_registry = FailedJobRegistry(connection=self.connection)
+        job = self.queue.enqueue(say_hello)
+
+        self.connection.zadd(self.registry.key, {f'{job.id}:execution_id': 100})
+
+        # Job has not been moved to FailedJobRegistry
+        self.registry.cleanup(1)
+        self.assertNotIn(job, failed_job_registry)
+        self.assertIn(job, self.registry)
+
+        with mock.patch.object(Job, 'execute_failure_callback') as mocked:
+            mock_handler = mock.MagicMock()
+            mock_handler.return_value = False
+            mock_handler_no_return = mock.MagicMock()
+            mock_handler_no_return.return_value = None
+            self.registry.cleanup(exception_handlers=[mock_handler_no_return, mock_handler])
+            mocked.assert_called_once_with(self.queue.death_penalty_class, AbandonedJobError, ANY, ANY)
+            mock_handler.assert_called_once_with(job, AbandonedJobError, ANY, ANY)
+            mock_handler_no_return.assert_called_once_with(job, AbandonedJobError, ANY, ANY)
+        self.assertIn(job.id, failed_job_registry)
+        self.assertNotIn(job, self.registry)
+        job.refresh()
+        self.assertEqual(job.get_status(), JobStatus.FAILED)
+        if job.supports_redis_streams:
+            latest_result = job.latest_result()
+            self.assertIsNotNone(latest_result)
+            self.assertTrue(latest_result.exc_string)  # explanation is written to exc_info
+        else:
+            self.assertTrue(job.exc_info)
+
+    def test_enqueue_dependents_when_parent_job_is_abandoned(self):
+        """Enqueuing parent job's dependencies after moving it to FailedJobRegistry due to AbandonedJobError."""
+        queue = Queue(connection=self.connection)
+        worker = Worker([queue])
+        failed_job_registry = FailedJobRegistry(connection=self.connection)
+        finished_job_registry = FinishedJobRegistry(connection=self.connection)
+        deferred_job_registry = DeferredJobRegistry(connection=self.connection)
+
+        parent_job = queue.enqueue(say_hello)
+        job_to_be_executed = queue.enqueue_call(say_hello, depends_on=Dependency(jobs=[parent_job], allow_failure=True))
+        job_not_to_be_executed = queue.enqueue_call(
+            say_hello, depends_on=Dependency(jobs=[parent_job], allow_failure=False)
+        )
+        self.assertIn(job_to_be_executed, deferred_job_registry)
+        self.assertIn(job_not_to_be_executed, deferred_job_registry)
+
+        self.connection.zadd(self.registry.key, {f'{parent_job.id}:execution': 2})
+        queue.remove(parent_job.id)
+
+        with mock.patch.object(Job, 'execute_failure_callback') as mocked:
+            self.registry.cleanup()
+            mocked.assert_called_once_with(queue.death_penalty_class, AbandonedJobError, ANY, ANY)
+
+        # check that parent job was moved to FailedJobRegistry and has correct status
+        self.assertIn(parent_job, failed_job_registry)
+        self.assertNotIn(parent_job, self.registry)
+        self.assertTrue(parent_job.is_failed)
+
+        # check that only job_to_be_executed has been queued and executed
+        self.assertEqual(len(queue.get_job_ids()), 1)
+        self.assertTrue(job_to_be_executed.is_queued)
+        self.assertFalse(job_not_to_be_executed.is_queued)
+
+        worker.work(burst=True)
+        self.assertTrue(job_to_be_executed.is_finished)
+        self.assertNotIn(job_to_be_executed, deferred_job_registry)
+        self.assertIn(job_to_be_executed, finished_job_registry)
+
+        self.assertFalse(job_not_to_be_executed.is_finished)
+        self.assertNotIn(job_not_to_be_executed, finished_job_registry)
+
+    def test_warnings_on_add_remove_and_exception(self):
+        """Test backwards compatibility of the .add and .remove methods for
+        the StartedJobRegistry."""
+        with pytest.raises(NotImplementedError):
+            self.registry.add('job_id')
+
+        with pytest.raises(NotImplementedError):
+            self.registry.remove('job_id')
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 7f2d338b3..057d36c9c 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -3,6 +3,7 @@
 from multiprocessing import Process
 from unittest import mock
 
+import pytest
 import redis
 
 from rq import Queue
@@ -130,6 +131,24 @@ def test_schedule(self):
                 self.connection.zscore(registry.key, job.id), 1546300800
             )  # 2019-01-01 UTC in Unix timestamp
 
+    def test_remove_jobs(self):
+        """Removing job ids from ScheduledJobRegistry. Will be deprecated in the future."""
+        queue = Queue(connection=self.connection)
+        registry = ScheduledJobRegistry(queue=queue)
+        timestamp = current_timestamp()
+
+        self.connection.zadd(registry.key, {'foo': 1})
+        self.connection.zadd(registry.key, {'bar': timestamp + 10})
+        self.connection.zadd(registry.key, {'baz': timestamp + 30})
+
+        with pytest.deprecated_call():
+            # without timestamp it should remove everything till the current timestamp
+            registry.remove_jobs()
+            self.assertListEqual(registry.get_jobs_to_schedule(timestamp=timestamp + 100), ['bar', 'baz'])
+            registry.remove_jobs(timestamp=timestamp + 15)
+            # should remove bar job
+            self.assertListEqual(registry.get_jobs_to_schedule(timestamp=timestamp + 100), ['baz'])
+
 
 class TestScheduler(RQTestCase):
     def test_init(self):
diff --git a/tests/test_worker.py b/tests/test_worker.py
index ebc76c2d8..11975676e 100644
--- a/tests/test_worker.py
+++ b/tests/test_worker.py
@@ -886,8 +886,8 @@ def test_prepare_job_execution(self):
 
         # Updates working queue, job execution should be there
         registry = StartedJobRegistry(connection=self.connection)
-        # self.assertTrue(job.id in registry.get_job_ids())
-        self.assertTrue(worker.execution.composite_key in registry.get_job_ids())
+        self.assertTrue(job.id in registry.get_job_ids())
+        self.assertTrue((worker.execution.job_id, worker.execution.id) in registry.get_job_and_execution_ids())
 
         # Updates worker's current job
         self.assertEqual(worker.get_current_job_id(), job.id)


Reviews:
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-16T01:51:03Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-16T01:54:03Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-16T01:56:16Z
user: login: OlegZv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-16T03:39:33Z
user: login: OlegZv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-16T03:43:41Z
user: login: OlegZv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-16T03:45:31Z
user: login: OlegZv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-16T04:06:22Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T01:25:49Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T01:30:14Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T01:32:26Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T01:35:07Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T01:36:40Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T01:42:22Z
user: login: OlegZv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T01:52:31Z
user: login: OlegZv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T02:30:16Z
user: login: OlegZv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T02:33:35Z
user: login: selwin
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T03:04:22Z
user: login: OlegZv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T03:08:13Z

Review comments:
diff_hunk: @@ -286,59 +296,138 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
         return job_ids
 
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Union[Execution, str]',
path: rq/registry.py
user: login: selwin
body: Not sure why `remove_execution()` should also accept `str`. I think `remove_execution` is only called with `execution` as argument.
created_at: 2025-03-16T01:51:03Z
author_association: COLLABORATOR
diff_hunk: @@ -286,59 +296,138 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
         return job_ids
 
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Union[Execution, str]',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ):
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Union[Execution,str]): The Execution to add
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        to_delete = execution if isinstance(execution, str) else execution.composite_key
+        result = connection.zrem(self.key, to_delete)
         # if delete_job:
         #     job.delete()
         return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Returns list of all job ids.
path: rq/registry.py
user: login: selwin
body: Can you clarify the doc string? This returns a list of tuple of job and execution ID.

In the future, I may add a dataclass to represent this structure too, to improve readability.
created_at: 2025-03-16T01:54:03Z
author_association: COLLABORATOR
diff_hunk: @@ -286,59 +296,138 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
         return job_ids
 
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Union[Execution, str]',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ):
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Union[Execution,str]): The Execution to add
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        to_delete = execution if isinstance(execution, str) else execution.composite_key
+        result = connection.zrem(self.key, to_delete)
         # if delete_job:
         #     job.delete()
         return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Returns list of all job ids.
+
+        Args:
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
+
+        Returns:
+            List[Tuple[str, str]]: list of the job ids in the registry
+        """
+        if cleanup:
+            self.cleanup()
+        return [
+            parse_composite_key(as_text(entry)) for entry in self.connection.zrange(self.key, start, end, desc=desc)
+        ]
+
+    def __contains__(self, item: Any) -> bool:
+        """Method to check if the item is in the registry.
+
+        Args:
+            item (Any): Either a Job (instance of job_class) or a job id.
+
+        Returns:
+            bool: True if the item is in the registry.
+        """
+        job_id = item
+        if isinstance(item, self.job_class):
+            job_id = item.id
+        return cast(str, job_id) in self.get_job_ids(cleanup=False)
+
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use add_execution method.', DeprecationWarning
+        )
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # use job_id: so the stored record is always a tuple, when split by ':'
+        return super().add(f'{job_id}:', ttl, pipeline, xx)
+
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use remove_execution method.', DeprecationWarning
+        )
+
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # if delete job is True, the following would fail. Prohibit deleting the job this way.
+        if delete_job:
+            raise RuntimeError('StartedJobRegistry.remove() method cannot auto-delete the job.')
+        return super().remove(f'{job_id}:', pipeline, delete_job)
+
+    @staticmethod
+    def parse_job_id(entry: str) -> str:
+        # other classes might have a different entry format
+        # base registry stores just the job id
+        if not isinstance(entry, str):
+            entry = as_text(entry)  # type: ignore
+        job_id, _execution_id = parse_composite_key(entry)
+        return job_id
+
+    def remove_executions(self, job: Job, pipeline: Optional['Pipeline'] = None):
path: rq/registry.py
user: login: selwin
body: Do you mind moving this method to `job.remove_executions()`? It's cleaner this way. 
created_at: 2025-03-16T01:56:16Z
author_association: COLLABORATOR
diff_hunk: @@ -286,59 +296,138 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
         return job_ids
 
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Union[Execution, str]',
path: rq/registry.py
user: login: OlegZv
body: Hmm... Now I'm trying to remember... I think originally I had some other code that just passed the `composite_key` instead of the execution object. This is because the function doesn't depend on the whole `Execution` object, but on the `composite_key`. **The Single Responsibility Principle**.
(Later I realized that this would break backwards compatibility, so I made it a `Union`. I think I'll return it back to the way it was)
created_at: 2025-03-16T03:39:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -286,59 +296,138 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
         return job_ids
 
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Union[Execution, str]',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ):
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Union[Execution,str]): The Execution to add
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        to_delete = execution if isinstance(execution, str) else execution.composite_key
+        result = connection.zrem(self.key, to_delete)
         # if delete_job:
         #     job.delete()
         return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Returns list of all job ids.
path: rq/registry.py
user: login: OlegZv
body: Clarified, good catch. I like the dataclass idea!
created_at: 2025-03-16T03:43:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -286,59 +296,138 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
         return job_ids
 
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Union[Execution, str]',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ):
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Union[Execution,str]): The Execution to add
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        to_delete = execution if isinstance(execution, str) else execution.composite_key
+        result = connection.zrem(self.key, to_delete)
         # if delete_job:
         #     job.delete()
         return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Returns list of all job ids.
+
+        Args:
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
+
+        Returns:
+            List[Tuple[str, str]]: list of the job ids in the registry
+        """
+        if cleanup:
+            self.cleanup()
+        return [
+            parse_composite_key(as_text(entry)) for entry in self.connection.zrange(self.key, start, end, desc=desc)
+        ]
+
+    def __contains__(self, item: Any) -> bool:
+        """Method to check if the item is in the registry.
+
+        Args:
+            item (Any): Either a Job (instance of job_class) or a job id.
+
+        Returns:
+            bool: True if the item is in the registry.
+        """
+        job_id = item
+        if isinstance(item, self.job_class):
+            job_id = item.id
+        return cast(str, job_id) in self.get_job_ids(cleanup=False)
+
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use add_execution method.', DeprecationWarning
+        )
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # use job_id: so the stored record is always a tuple, when split by ':'
+        return super().add(f'{job_id}:', ttl, pipeline, xx)
+
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use remove_execution method.', DeprecationWarning
+        )
+
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # if delete job is True, the following would fail. Prohibit deleting the job this way.
+        if delete_job:
+            raise RuntimeError('StartedJobRegistry.remove() method cannot auto-delete the job.')
+        return super().remove(f'{job_id}:', pipeline, delete_job)
+
+    @staticmethod
+    def parse_job_id(entry: str) -> str:
+        # other classes might have a different entry format
+        # base registry stores just the job id
+        if not isinstance(entry, str):
+            entry = as_text(entry)  # type: ignore
+        job_id, _execution_id = parse_composite_key(entry)
+        return job_id
+
+    def remove_executions(self, job: Job, pipeline: Optional['Pipeline'] = None):
path: rq/registry.py
user: login: OlegZv
body: I can totally do that. Would that be a breaking change though?
Maybe for now I can leave the registry method with a deprecation warning and inside just call `job.remove_executions()`?
created_at: 2025-03-16T03:45:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -286,59 +296,138 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
         return job_ids
 
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Union[Execution, str]',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ):
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Union[Execution,str]): The Execution to add
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        to_delete = execution if isinstance(execution, str) else execution.composite_key
+        result = connection.zrem(self.key, to_delete)
         # if delete_job:
         #     job.delete()
         return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Returns list of all job ids.
+
+        Args:
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
+
+        Returns:
+            List[Tuple[str, str]]: list of the job ids in the registry
+        """
+        if cleanup:
+            self.cleanup()
+        return [
+            parse_composite_key(as_text(entry)) for entry in self.connection.zrange(self.key, start, end, desc=desc)
+        ]
+
+    def __contains__(self, item: Any) -> bool:
+        """Method to check if the item is in the registry.
+
+        Args:
+            item (Any): Either a Job (instance of job_class) or a job id.
+
+        Returns:
+            bool: True if the item is in the registry.
+        """
+        job_id = item
+        if isinstance(item, self.job_class):
+            job_id = item.id
+        return cast(str, job_id) in self.get_job_ids(cleanup=False)
+
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use add_execution method.', DeprecationWarning
+        )
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # use job_id: so the stored record is always a tuple, when split by ':'
+        return super().add(f'{job_id}:', ttl, pipeline, xx)
+
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use remove_execution method.', DeprecationWarning
+        )
+
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # if delete job is True, the following would fail. Prohibit deleting the job this way.
+        if delete_job:
+            raise RuntimeError('StartedJobRegistry.remove() method cannot auto-delete the job.')
+        return super().remove(f'{job_id}:', pipeline, delete_job)
+
+    @staticmethod
+    def parse_job_id(entry: str) -> str:
+        # other classes might have a different entry format
+        # base registry stores just the job id
+        if not isinstance(entry, str):
+            entry = as_text(entry)  # type: ignore
+        job_id, _execution_id = parse_composite_key(entry)
+        return job_id
+
+    def remove_executions(self, job: Job, pipeline: Optional['Pipeline'] = None):
path: rq/registry.py
user: login: OlegZv
body: I made the change, and also noticed `def cleanup_execution()` for the base worker uses the deprecated `.remove` for the started_job_registry. I updated it as well. 
created_at: 2025-03-16T04:06:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1677,6 +1679,17 @@ def dependencies_are_met(
 
         return all(status.decode() in allowed_statuses for status in dependencies_statuses if status)
 
+    def remove_executions(self, pipeline: Optional['Pipeline'] = None) -> None:
path: rq/job.py
user: login: selwin
body: ~~It seems like this method is not needed. Mind deleting this?~~ Never mind, Github was hiding `registry.py` changes from me.
created_at: 2025-03-19T01:25:49Z
author_association: COLLABORATOR
diff_hunk: @@ -286,59 +296,136 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
-        return job_ids
-
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Execution',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ) -> None:
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Execution): The Execution to remove
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        connection.zrem(self.key, execution.composite_key)
         # if delete_job:
         #     job.delete()
-        return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Function to retrieve a list of tuples where the first item is the job id and
+            the second is the execution id.
+
+        Args:
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
+
+        Returns:
+            List[Tuple[str, str]]: a list of tuples where the first item is the job id and
+            the second is the execution id.
+        """
+        if cleanup:
+            self.cleanup()
+        return [
+            parse_composite_key(as_text(entry)) for entry in self.connection.zrange(self.key, start, end, desc=desc)
+        ]
+
+    def __contains__(self, item: Any) -> bool:
+        """Method to check if the item is in the registry.
+
+        Args:
+            item (Any): Either a Job (instance of job_class) or a job id.
+
+        Returns:
+            bool: True if the item is in the registry.
+        """
+        job_id = item
+        if isinstance(item, self.job_class):
+            job_id = item.id
+        return cast(str, job_id) in self.get_job_ids(cleanup=False)
+
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use add_execution method.', DeprecationWarning
+        )
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # use job_id: so the stored record is always a tuple, when split by ':'
+        return super().add(f'{job_id}:', ttl, pipeline, xx)
+
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
path: rq/registry.py
user: login: selwin
body: I think it's safe to change `add()` and `remove()` methods in `StartedJobRegistry` to raise `NotImplementedError`. The fact that they're still present is a bug, they should not be used in RQ >= 2.
created_at: 2025-03-19T01:30:14Z
author_association: COLLABORATOR
diff_hunk: @@ -286,59 +296,136 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
-        return job_ids
-
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Execution',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ) -> None:
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Execution): The Execution to remove
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        connection.zrem(self.key, execution.composite_key)
         # if delete_job:
         #     job.delete()
-        return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Function to retrieve a list of tuples where the first item is the job id and
+            the second is the execution id.
+
+        Args:
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
+
+        Returns:
+            List[Tuple[str, str]]: a list of tuples where the first item is the job id and
+            the second is the execution id.
+        """
+        if cleanup:
+            self.cleanup()
+        return [
+            parse_composite_key(as_text(entry)) for entry in self.connection.zrange(self.key, start, end, desc=desc)
+        ]
+
+    def __contains__(self, item: Any) -> bool:
+        """Method to check if the item is in the registry.
+
+        Args:
+            item (Any): Either a Job (instance of job_class) or a job id.
+
+        Returns:
+            bool: True if the item is in the registry.
+        """
+        job_id = item
+        if isinstance(item, self.job_class):
+            job_id = item.id
+        return cast(str, job_id) in self.get_job_ids(cleanup=False)
+
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use add_execution method.', DeprecationWarning
+        )
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # use job_id: so the stored record is always a tuple, when split by ':'
+        return super().add(f'{job_id}:', ttl, pipeline, xx)
+
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use remove_execution method.', DeprecationWarning
+        )
+
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # if delete job is True, the following would fail. Prohibit deleting the job this way.
+        if delete_job:
+            raise RuntimeError('StartedJobRegistry.remove() method cannot auto-delete the job.')
+        return super().remove(f'{job_id}:', pipeline, delete_job)
+
+    @staticmethod
+    def parse_job_id(entry: str) -> str:
+        # other classes might have a different entry format
+        # base registry stores just the job id
+        if not isinstance(entry, str):
+            entry = as_text(entry)  # type: ignore
+        job_id, _execution_id = parse_composite_key(entry)
+        return job_id
+
+    def remove_executions(self, job: Job, pipeline: Optional['Pipeline'] = None) -> None:
path: rq/registry.py
user: login: selwin
body: Can we delete this? Having a function that's calling another function is adding an unnecessary level of indirection.
created_at: 2025-03-19T01:32:26Z
author_association: COLLABORATOR
diff_hunk: @@ -9,13 +11,15 @@
 
 
 class TestRegistry(RQTestCase):
+    """Test the execution registry."""
+
     def setUp(self):
         super().setUp()
+        self.q = Queue(connection=self.connection)
path: tests/test_executions.py
user: login: selwin
body: Nitpick: mind changing `self.q` to `self.queue`? I want to keep things consistent whenever possible.
created_at: 2025-03-19T01:35:07Z
author_association: COLLABORATOR
diff_hunk: @@ -653,3 +517,227 @@ def test_worker_handle_job_failure(self):
         job = q.enqueue(div_by_zero, failure_ttl=5)
         w.handle_job_failure(job, q)
         self.assertLess(self.connection.zscore(registry.key, job.id), timestamp + 7)
+
+
+class TestStartedJobRegistry(RQTestCase):
+    def setUp(self):
+        super().setUp()
+        self.registry = StartedJobRegistry(connection=self.connection)
+        self.q = Queue(connection=self.connection)
path: tests/test_registry.py
user: login: selwin
body: `self.queue`
created_at: 2025-03-19T01:36:40Z
author_association: COLLABORATOR
diff_hunk: @@ -1677,6 +1679,17 @@ def dependencies_are_met(
 
         return all(status.decode() in allowed_statuses for status in dependencies_statuses if status)
 
+    def remove_executions(self, pipeline: Optional['Pipeline'] = None) -> None:
path: rq/job.py
user: login: selwin
body: I'm really sorry about this, it seems like I was mistaken in reading the diffs from the previous PR. `started_job_registry.remove_executions()` was an existing method, right? In that case, there's no need to remove this method to job class, there's no point in moving things around without a good reason.
created_at: 2025-03-19T01:42:21Z
author_association: COLLABORATOR
diff_hunk: @@ -286,59 +296,136 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
-        return job_ids
-
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Execution',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ) -> None:
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Execution): The Execution to remove
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        connection.zrem(self.key, execution.composite_key)
         # if delete_job:
         #     job.delete()
-        return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Function to retrieve a list of tuples where the first item is the job id and
+            the second is the execution id.
+
+        Args:
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
+
+        Returns:
+            List[Tuple[str, str]]: a list of tuples where the first item is the job id and
+            the second is the execution id.
+        """
+        if cleanup:
+            self.cleanup()
+        return [
+            parse_composite_key(as_text(entry)) for entry in self.connection.zrange(self.key, start, end, desc=desc)
+        ]
+
+    def __contains__(self, item: Any) -> bool:
+        """Method to check if the item is in the registry.
+
+        Args:
+            item (Any): Either a Job (instance of job_class) or a job id.
+
+        Returns:
+            bool: True if the item is in the registry.
+        """
+        job_id = item
+        if isinstance(item, self.job_class):
+            job_id = item.id
+        return cast(str, job_id) in self.get_job_ids(cleanup=False)
+
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use add_execution method.', DeprecationWarning
+        )
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # use job_id: so the stored record is always a tuple, when split by ':'
+        return super().add(f'{job_id}:', ttl, pipeline, xx)
+
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
path: rq/registry.py
user: login: OlegZv
body: Roger that! will add NotImplemented.
created_at: 2025-03-19T01:52:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -286,59 +296,136 @@ def cleanup(self, timestamp: Optional[float] = None, exception_handlers: Optiona
 
                     if retry:
                         job.retry(queue, pipeline)
-
                     else:
-                        exc_string = f'due to {AbandonedJobError.__name__}'
-                        logger.warning(
-                            f'{self.__class__.__name__} cleanup: Moving job {job.id} to {FailedJobRegistry.__name__} '
-                            f'({exc_string})'
+                        exc_string = (
+                            f'Moved to {FailedJobRegistry.__name__}, due to {AbandonedJobError.__name__}, at {now()}'
                         )
-                        job.set_status(JobStatus.FAILED)
-                        job._exc_info = f'Moved to {FailedJobRegistry.__name__}, {exc_string}, at {datetime.now()}'
-                        job.save(pipeline=pipeline, include_meta=False)
-                        job.cleanup(ttl=-1, pipeline=pipeline)
-                        failed_job_registry.add(job, job.failure_ttl)
-                        queue.enqueue_dependents(job)
+                        logger.warning('%s cleanup: %s %s', self.__class__.__name__, job.id, exc_string)
+                        job.set_status(JobStatus.FAILED, pipeline=pipeline)
+                        job._handle_failure(exc_string, pipeline)
+                        # don't refresh the job status, because the job state is still in the pipeline
+                        queue.enqueue_dependents(job, refresh_job_status=False)
 
                 pipeline.zremrangebyscore(self.key, 0, score)
                 pipeline.execute()
 
-        return job_ids
-
-    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl=0, xx: bool = False) -> int:
+    def add_execution(self, execution: 'Execution', pipeline: 'Pipeline', ttl: int = 0, xx: bool = False) -> int:
         """Adds an execution to a registry with expiry time of now + ttl, unless it's -1 which is set to +inf
 
         Args:
-            execution (Execution): The Execution to add
+            execution (Execution): The Execution to add.
+            pipeline (Pipeline): The Redis Pipeline.
             ttl (int, optional): The time to live. Defaults to 0.
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
             xx (bool, optional): .... Defaults to False.
 
         Returns:
             result (int): The ZADD command result
         """
-        score = ttl if ttl < 0 else current_timestamp() + ttl
+        score: Union[int, str] = ttl if ttl < 0 else current_timestamp() + ttl
         if score == -1:
             score = '+inf'
 
         return pipeline.zadd(self.key, {execution.composite_key: score}, xx=xx)  # type: ignore
 
-    def remove_execution(self, execution: 'Execution', job: 'Job', pipeline: 'Pipeline', delete_job: bool = False):
+    def remove_execution(
+        self,
+        execution: 'Execution',
+        job: Optional[Job] = None,
+        pipeline: Optional['Pipeline'] = None,
+        delete_job: bool = False,
+    ) -> None:
         """Removes job from registry and deletes it if `delete_job == True`
 
         Args:
-            execution (Execution): The Execution to add
-            job (Job): The Job to remove from the registry
-            pipeline (Optional[Pipeline], optional): The Redis Pipeline. Defaults to None.
+            execution (Execution): The Execution to remove
+            job (Optional[Job]): The Job to remove from the registry
+            pipeline (Optional['Pipeline'], optional): The Redis Pipeline. Defaults to None.
             delete_job (bool, optional): If should delete the job.. Defaults to False.
         """
         connection = pipeline if pipeline is not None else self.connection
-        result = connection.zrem(self.key, execution.composite_key)
+        connection.zrem(self.key, execution.composite_key)
         # if delete_job:
         #     job.delete()
-        return result
 
-    # TODO: needs to add a method to cleanup executions
+    def get_job_and_execution_ids(
+        self, start: int = 0, end: int = -1, desc: bool = False, cleanup: bool = True
+    ) -> List[Tuple[str, str]]:
+        """Function to retrieve a list of tuples where the first item is the job id and
+            the second is the execution id.
+
+        Args:
+            start (int, optional): start rank. Defaults to 0.
+            end (int, optional): end rank. Defaults to -1.
+            desc (bool, optional): sort in reversed order. Defaults to False.
+            cleanup (bool, optional): whether to perform the cleanup. Defaults to True.
+
+        Returns:
+            List[Tuple[str, str]]: a list of tuples where the first item is the job id and
+            the second is the execution id.
+        """
+        if cleanup:
+            self.cleanup()
+        return [
+            parse_composite_key(as_text(entry)) for entry in self.connection.zrange(self.key, start, end, desc=desc)
+        ]
+
+    def __contains__(self, item: Any) -> bool:
+        """Method to check if the item is in the registry.
+
+        Args:
+            item (Any): Either a Job (instance of job_class) or a job id.
+
+        Returns:
+            bool: True if the item is in the registry.
+        """
+        job_id = item
+        if isinstance(item, self.job_class):
+            job_id = item.id
+        return cast(str, job_id) in self.get_job_ids(cleanup=False)
+
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use add_execution method.', DeprecationWarning
+        )
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # use job_id: so the stored record is always a tuple, when split by ':'
+        return super().add(f'{job_id}:', ttl, pipeline, xx)
+
+    def remove(self, job: Union[Job, str], pipeline: Optional['Pipeline'] = None, delete_job: bool = False):
+        warnings.warn(
+            'StartedJobRegistry can only contain execution records. Use remove_execution method.', DeprecationWarning
+        )
+
+        # in the future will raise
+        # raise NotImplementedError('')
+        job_id = job.id if isinstance(job, self.job_class) else job
+        # if delete job is True, the following would fail. Prohibit deleting the job this way.
+        if delete_job:
+            raise RuntimeError('StartedJobRegistry.remove() method cannot auto-delete the job.')
+        return super().remove(f'{job_id}:', pipeline, delete_job)
+
+    @staticmethod
+    def parse_job_id(entry: str) -> str:
+        # other classes might have a different entry format
+        # base registry stores just the job id
+        if not isinstance(entry, str):
+            entry = as_text(entry)  # type: ignore
+        job_id, _execution_id = parse_composite_key(entry)
+        return job_id
+
+    def remove_executions(self, job: Job, pipeline: Optional['Pipeline'] = None) -> None:
path: rq/registry.py
user: login: OlegZv
body: ~~Sure thing. I wanted to maintain backward compatibility **in case anyone is using this method**. Let's make sure we call this out in the release notes.~~
created_at: 2025-03-19T02:30:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1677,6 +1679,17 @@ def dependencies_are_met(
 
         return all(status.decode() in allowed_statuses for status in dependencies_statuses if status)
 
+    def remove_executions(self, pipeline: Optional['Pipeline'] = None) -> None:
path: rq/job.py
user: login: OlegZv
body: I wish I saw this comment before removing this method :)
All good! I'll leave remove it from the `Job` class and keep it in the `StartedJobRegistry`.
created_at: 2025-03-19T02:33:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -103,32 +103,33 @@ def get_job_count(self, cleanup=True) -> int:
             self.cleanup()
         return self.connection.zcard(self.key)
 
-    def add(self, job: 'Job', ttl=0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
path: rq/registry.py
user: login: selwin
body: Just want to make sure that adding `str` type is necessary.

Otherwise I think this PR looks good.
created_at: 2025-03-19T03:04:22Z
author_association: COLLABORATOR
diff_hunk: @@ -103,32 +103,33 @@ def get_job_count(self, cleanup=True) -> int:
             self.cleanup()
         return self.connection.zcard(self.key)
 
-    def add(self, job: 'Job', ttl=0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
+    def add(self, job: Union[Job, str], ttl: int = 0, pipeline: Optional['Pipeline'] = None, xx: bool = False) -> int:
path: rq/registry.py
user: login: OlegZv
body: Updated!
created_at: 2025-03-19T03:08:13Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov[bot]
created_at: 2025-03-15T18:58:27Z
body: ## [Codecov](https://app.codecov.io/gh/rq/rq/pull/2208?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=rq) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 93.69%. Comparing base [(`2de9491`)](https://app.codecov.io/gh/rq/rq/commit/2de949126fb0213c3654051d49fafc84e1e54193?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=rq) to head [(`20a33f1`)](https://app.codecov.io/gh/rq/rq/commit/20a33f1281faf5a37bf95a0840169becb474c264?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=rq).
> Report is 129 commits behind head on master.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #2208      +/-   ##
==========================================
+ Coverage   93.61%   93.69%   +0.08%     
==========================================
  Files          28       30       +2     
  Lines        3760     4141     +381     
==========================================
+ Hits         3520     3880     +360     
- Misses        240      261      +21     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/rq/rq/pull/2208?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=rq).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=rq).

<details><summary>🚀 New features to boost your workflow: </summary>

- ❄ [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: OlegZv
created_at: 2025-03-18T23:38:15Z
body: @selwin do you think you'll be able to take another look this week? Next week might be busy for me and I know you wanted to push and release this soon.
author_association: CONTRIBUTOR
user: login: selwin
created_at: 2025-03-20T10:43:59Z
body: Thanks!
author_association: COLLABORATOR