Repository: apache/airflow

Issue:
Amazon NeptuneStopDbClusterOperator bug
### Apache Airflow Provider(s)

amazon

### Versions of Apache Airflow Providers

latest; AWS System test stack pulls from main

### Apache Airflow version

latest; AWS System test stack pulls from main

### Operating System

linux

### Deployment

Docker-Compose

### Deployment details

_No response_

### What happened

On occasion the stop_db_cluster() call will return a "backing-up" state which is not handled, causing an exception.  Example log message:

```
ERROR [airflow.task] Task failed with exception
Traceback (most recent call last):
  File "/opt/airflow/airflow/models/taskinstance.py", line 447, in _execute_task
    result = _execute_callable(context=context, **execute_callable_kwargs)
  File "/opt/airflow/airflow/models/taskinstance.py", line 417, in _execute_callable
    return execute_callable(context=context, **execute_callable_kwargs)
  File "/opt/airflow/airflow/providers/amazon/aws/operators/neptune.py", line 186, in execute
    resp = self.hook.conn.stop_db_cluster(DBClusterIdentifier=self.cluster_id)
  File "/usr/local/lib/python3.8/site-packages/botocore/client.py", line 553, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python3.8/site-packages/botocore/client.py", line 1009, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.errorfactory.InvalidDBClusterStateFault: An error occurred (InvalidDBClusterStateFault) when calling the StopDBCluster operation: DbCluster env989356b0-cluster is in backing-up state but expected it to be one of available.
INFO  [airflow.models.taskinstance] Marking task as UP_FOR_RETRY. dag_id=example_neptune, task_id=stop_task, execution_date=20210101T000000, start_date=20240313T100059, end_date=20240313T100100
```

### What you think should happen instead

If that is a possible state, it needs to be handled gracefully

### How to reproduce

The issue is intermittent so I'm not 100% sure.  Force a cluster to start a backup then immediately try to delete it while it is in progress.

### Anything else

_No response_

### Are you willing to submit PR?

- [X] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)



Discussion:

Resolving patch:
diff --git a/airflow/providers/amazon/aws/hooks/neptune.py b/airflow/providers/amazon/aws/hooks/neptune.py
--- a/airflow/providers/amazon/aws/hooks/neptune.py
+++ b/airflow/providers/amazon/aws/hooks/neptune.py
@@ -34,6 +34,12 @@ class NeptuneHook(AwsBaseHook):
 
     AVAILABLE_STATES = ["available"]
     STOPPED_STATES = ["stopped"]
+    ERROR_STATES = [
+        "cloning-failed",
+        "inaccessible-encryption-credentials",
+        "inaccessible-encryption-credentials-recoverable",
+        "migration-failed",
+    ]
 
     def __init__(self, *args, **kwargs):
         kwargs["client_type"] = "neptune"
@@ -82,4 +88,33 @@ def get_cluster_status(self, cluster_id: str) -> str:
         :param cluster_id: The ID of the cluster to get the status of.
         :return: The status of the cluster.
         """
-        return self.get_conn().describe_db_clusters(DBClusterIdentifier=cluster_id)["DBClusters"][0]["Status"]
+        return self.conn.describe_db_clusters(DBClusterIdentifier=cluster_id)["DBClusters"][0]["Status"]
+
+    def get_db_instance_status(self, instance_id: str) -> str:
+        """
+        Get the status of a Neptune instance.
+
+        :param instance_id: The ID of the instance to get the status of.
+        :return: The status of the instance.
+        """
+        return self.conn.describe_db_instances(DBInstanceIdentifier=instance_id)["DBInstances"][0][
+            "DBInstanceStatus"
+        ]
+
+    def wait_for_cluster_instance_availability(
+        self, cluster_id: str, delay: int = 30, max_attempts: int = 60
+    ) -> None:
+        """
+        Wait for Neptune instances in a cluster to be available.
+
+        :param cluster_id: The cluster ID of the instances to wait for.
+        :param delay: Time in seconds to delay between polls.
+        :param max_attempts: Maximum number of attempts to poll for completion.
+        :return: The status of the instances.
+        """
+        filters = [{"Name": "db-cluster-id", "Values": [cluster_id]}]
+        self.log.info("Waiting for instances in cluster %s.", cluster_id)
+        self.get_waiter("db_instance_available").wait(
+            Filters=filters, WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}
+        )
+        self.log.info("Finished waiting for instances in cluster %s.", cluster_id)
diff --git a/airflow/providers/amazon/aws/operators/neptune.py b/airflow/providers/amazon/aws/operators/neptune.py
--- a/airflow/providers/amazon/aws/operators/neptune.py
+++ b/airflow/providers/amazon/aws/operators/neptune.py
@@ -19,11 +19,15 @@
 
 from typing import TYPE_CHECKING, Any, Sequence
 
+from botocore.exceptions import ClientError
+
 from airflow.configuration import conf
+from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.neptune import NeptuneHook
 from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
 from airflow.providers.amazon.aws.triggers.neptune import (
     NeptuneClusterAvailableTrigger,
+    NeptuneClusterInstancesAvailableTrigger,
     NeptuneClusterStoppedTrigger,
 )
 from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
@@ -32,6 +36,50 @@
     from airflow.utils.context import Context
 
 
+def handle_waitable_exception(
+    operator: NeptuneStartDbClusterOperator | NeptuneStopDbClusterOperator, err: str
+):
+    """
+    Handle client exceptions for invalid cluster or invalid instance status that are temporary.
+
+    After status change, it's possible to retry. Waiter will handle terminal status.
+    """
+    code = err
+
+    if code in ("InvalidDBInstanceStateFault", "InvalidDBInstanceState"):
+        if operator.deferrable:
+            operator.log.info("Deferring until instances become available: %s", operator.cluster_id)
+            operator.defer(
+                trigger=NeptuneClusterInstancesAvailableTrigger(
+                    aws_conn_id=operator.aws_conn_id,
+                    db_cluster_id=operator.cluster_id,
+                    region_name=operator.region_name,
+                    botocore_config=operator.botocore_config,
+                    verify=operator.verify,
+                ),
+                method_name="execute",
+            )
+        else:
+            operator.log.info("Need to wait for instances to become available: %s", operator.cluster_id)
+            operator.hook.wait_for_cluster_instance_availability(cluster_id=operator.cluster_id)
+    if code in ["InvalidClusterState", "InvalidDBClusterStateFault"]:
+        if operator.deferrable:
+            operator.log.info("Deferring until cluster becomes available: %s", operator.cluster_id)
+            operator.defer(
+                trigger=NeptuneClusterAvailableTrigger(
+                    aws_conn_id=operator.aws_conn_id,
+                    db_cluster_id=operator.cluster_id,
+                    region_name=operator.region_name,
+                    botocore_config=operator.botocore_config,
+                    verify=operator.verify,
+                ),
+                method_name="execute",
+            )
+        else:
+            operator.log.info("Need to wait for cluster to become available: %s", operator.cluster_id)
+            operator.hook.wait_for_cluster_availability(operator.cluster_id)
+
+
 class NeptuneStartDbClusterOperator(AwsBaseOperator[NeptuneHook]):
     """Starts an Amazon Neptune DB cluster.
 
@@ -78,10 +126,10 @@ def __init__(
         self.cluster_id = db_cluster_id
         self.wait_for_completion = wait_for_completion
         self.deferrable = deferrable
-        self.delay = waiter_delay
-        self.max_attempts = waiter_max_attempts
+        self.waiter_delay = waiter_delay
+        self.waiter_max_attempts = waiter_max_attempts
 
-    def execute(self, context: Context) -> dict[str, str]:
+    def execute(self, context: Context, event: dict[str, Any] | None = None, **kwargs) -> dict[str, str]:
         self.log.info("Starting Neptune cluster: %s", self.cluster_id)
 
         # Check to make sure the cluster is not already available.
@@ -89,9 +137,32 @@ def execute(self, context: Context) -> dict[str, str]:
         if status.lower() in NeptuneHook.AVAILABLE_STATES:
             self.log.info("Neptune cluster %s is already available.", self.cluster_id)
             return {"db_cluster_id": self.cluster_id}
-
-        resp = self.hook.conn.start_db_cluster(DBClusterIdentifier=self.cluster_id)
-        status = resp.get("DBClusters", {}).get("Status", "Unknown")
+        elif status.lower() in NeptuneHook.ERROR_STATES:
+            # some states will not allow you to start the cluster
+            self.log.error(
+                "Neptune cluster %s is in error state %s and cannot be started", self.cluster_id, status
+            )
+            raise AirflowException(f"Neptune cluster {self.cluster_id} is in error state {status}")
+
+        """
+        A cluster and its instances must be in a valid state to send the start request.
+        This loop covers the case where the cluster is not available and also the case where
+        the cluster is available, but one or more of the instances are in an invalid state.
+        If either are in an invalid state, wait for the availability and retry.
+        Let the waiters handle retries and detecting the error states.
+        """
+        try:
+            self.hook.conn.start_db_cluster(DBClusterIdentifier=self.cluster_id)
+        except ClientError as ex:
+            code = ex.response["Error"]["Code"]
+            self.log.warning("Received client error when attempting to start the cluster: %s", code)
+
+            if code in ["InvalidDBInstanceState", "InvalidClusterState", "InvalidDBClusterStateFault"]:
+                handle_waitable_exception(operator=self, err=code)
+
+            else:
+                # re raise for any other type of client error
+                raise
 
         if self.deferrable:
             self.log.info("Deferring for cluster start: %s", self.cluster_id)
@@ -100,15 +171,17 @@ def execute(self, context: Context) -> dict[str, str]:
                 trigger=NeptuneClusterAvailableTrigger(
                     aws_conn_id=self.aws_conn_id,
                     db_cluster_id=self.cluster_id,
-                    waiter_delay=self.delay,
-                    waiter_max_attempts=self.max_attempts,
+                    waiter_delay=self.waiter_delay,
+                    waiter_max_attempts=self.waiter_max_attempts,
                 ),
                 method_name="execute_complete",
             )
 
         elif self.wait_for_completion:
             self.log.info("Waiting for Neptune cluster %s to start.", self.cluster_id)
-            self.hook.wait_for_cluster_availability(self.cluster_id, self.delay, self.max_attempts)
+            self.hook.wait_for_cluster_availability(
+                self.cluster_id, self.waiter_delay, self.waiter_max_attempts
+            )
 
         return {"db_cluster_id": self.cluster_id}
 
@@ -171,20 +244,53 @@ def __init__(
         self.cluster_id = db_cluster_id
         self.wait_for_completion = wait_for_completion
         self.deferrable = deferrable
-        self.delay = waiter_delay
-        self.max_attempts = waiter_max_attempts
+        self.waiter_delay = waiter_delay
+        self.waiter_max_attempts = waiter_max_attempts
 
-    def execute(self, context: Context) -> dict[str, str]:
+    def execute(self, context: Context, event: dict[str, Any] | None = None, **kwargs) -> dict[str, str]:
         self.log.info("Stopping Neptune cluster: %s", self.cluster_id)
 
-        # Check to make sure the cluster is not already stopped.
+        # Check to make sure the cluster is not already stopped or that its not in a bad state
         status = self.hook.get_cluster_status(self.cluster_id)
+        self.log.info("Current status: %s", status)
+
         if status.lower() in NeptuneHook.STOPPED_STATES:
             self.log.info("Neptune cluster %s is already stopped.", self.cluster_id)
             return {"db_cluster_id": self.cluster_id}
-
-        resp = self.hook.conn.stop_db_cluster(DBClusterIdentifier=self.cluster_id)
-        status = resp.get("DBClusters", {}).get("Status", "Unknown")
+        elif status.lower() in NeptuneHook.ERROR_STATES:
+            # some states will not allow you to stop the cluster
+            self.log.error(
+                "Neptune cluster %s is in error state %s and cannot be stopped", self.cluster_id, status
+            )
+            raise AirflowException(f"Neptune cluster {self.cluster_id} is in error state {status}")
+
+        """
+        A cluster and its instances must be in a valid state to send the stop request.
+        This loop covers the case where the cluster is not available and also the case where
+        the cluster is available, but one or more of the instances are in an invalid state.
+        If either are in an invalid state, wait for the availability and retry.
+        Let the waiters handle retries and detecting the error states.
+        """
+
+        try:
+            self.hook.conn.stop_db_cluster(DBClusterIdentifier=self.cluster_id)
+
+        # cluster must be in available state to stop it
+        except ClientError as ex:
+            code = ex.response["Error"]["Code"]
+            self.log.warning("Received client error when attempting to stop the cluster: %s", code)
+
+            # these can be handled by a waiter
+            if code in [
+                "InvalidDBInstanceState",
+                "InvalidDBInstanceStateFault",
+                "InvalidClusterState",
+                "InvalidDBClusterStateFault",
+            ]:
+                handle_waitable_exception(self, code)
+            else:
+                # re raise for any other type of client error
+                raise
 
         if self.deferrable:
             self.log.info("Deferring for cluster stop: %s", self.cluster_id)
@@ -193,22 +299,23 @@ def execute(self, context: Context) -> dict[str, str]:
                 trigger=NeptuneClusterStoppedTrigger(
                     aws_conn_id=self.aws_conn_id,
                     db_cluster_id=self.cluster_id,
-                    waiter_delay=self.delay,
-                    waiter_max_attempts=self.max_attempts,
+                    waiter_delay=self.waiter_delay,
+                    waiter_max_attempts=self.waiter_max_attempts,
                 ),
                 method_name="execute_complete",
             )
 
         elif self.wait_for_completion:
-            self.log.info("Waiting for Neptune cluster %s to start.", self.cluster_id)
-            self.hook.wait_for_cluster_stopped(self.cluster_id, self.delay, self.max_attempts)
+            self.log.info("Waiting for Neptune cluster %s to stop.", self.cluster_id)
+
+            self.hook.wait_for_cluster_stopped(self.cluster_id, self.waiter_delay, self.waiter_max_attempts)
 
         return {"db_cluster_id": self.cluster_id}
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, str]:
         status = ""
         cluster_id = ""
-
+        self.log.info(event)
         if event:
             status = event.get("status", "")
             cluster_id = event.get("cluster_id", "")
diff --git a/airflow/providers/amazon/aws/triggers/neptune.py b/airflow/providers/amazon/aws/triggers/neptune.py
--- a/airflow/providers/amazon/aws/triggers/neptune.py
+++ b/airflow/providers/amazon/aws/triggers/neptune.py
@@ -113,3 +113,48 @@ def hook(self) -> AwsGenericHook:
             verify=self.verify,
             config=self.botocore_config,
         )
+
+
+class NeptuneClusterInstancesAvailableTrigger(AwsBaseWaiterTrigger):
+    """
+    Triggers when a Neptune Cluster Instance is available.
+
+    :param db_cluster_id: Cluster ID to wait on instances from
+    :param waiter_delay: The amount of time in seconds to wait between attempts.
+    :param waiter_max_attempts: The maximum number of attempts to be made.
+    :param aws_conn_id: The Airflow connection used for AWS credentials.
+    :param region_name: AWS region name (example: us-east-1)
+    """
+
+    def __init__(
+        self,
+        *,
+        db_cluster_id: str,
+        waiter_delay: int = 30,
+        waiter_max_attempts: int = 60,
+        aws_conn_id: str | None = None,
+        region_name: str | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            serialized_fields={"db_cluster_id": db_cluster_id},
+            waiter_name="db_instance_available",
+            waiter_args={"Filters": [{"Name": "db-cluster-id", "Values": [db_cluster_id]}]},
+            failure_message="Failed to start Neptune instances",
+            status_message="Status of Neptune instances are",
+            status_queries=["DBInstances[].Status"],
+            return_key="db_cluster_id",
+            return_value=db_cluster_id,
+            waiter_delay=waiter_delay,
+            waiter_max_attempts=waiter_max_attempts,
+            aws_conn_id=aws_conn_id,
+            **kwargs,
+        )
+
+    def hook(self) -> AwsGenericHook:
+        return NeptuneHook(
+            aws_conn_id=self.aws_conn_id,
+            region_name=self.region_name,
+            verify=self.verify,
+            config=self.botocore_config,
+        )


Test patch:
diff --git a/tests/providers/amazon/aws/hooks/test_neptune.py b/tests/providers/amazon/aws/hooks/test_neptune.py
--- a/tests/providers/amazon/aws/hooks/test_neptune.py
+++ b/tests/providers/amazon/aws/hooks/test_neptune.py
@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 from typing import Generator
+from unittest import mock
 
 import pytest
 from moto import mock_aws
@@ -50,3 +51,10 @@ def test_get_conn_returns_a_boto3_connection(self):
 
     def test_get_cluster_status(self, neptune_hook: NeptuneHook, neptune_cluster_id):
         assert neptune_hook.get_cluster_status(neptune_cluster_id) is not None
+
+    @mock.patch.object(NeptuneHook, "get_waiter")
+    def test_wait_for_cluster_instance_availability(
+        self, mock_get_waiter, neptune_hook: NeptuneHook, neptune_cluster_id
+    ):
+        neptune_hook.wait_for_cluster_instance_availability(neptune_cluster_id)
+        mock_get_waiter.assert_called_once_with("db_instance_available")
diff --git a/tests/providers/amazon/aws/operators/test_neptune.py b/tests/providers/amazon/aws/operators/test_neptune.py
--- a/tests/providers/amazon/aws/operators/test_neptune.py
+++ b/tests/providers/amazon/aws/operators/test_neptune.py
@@ -21,8 +21,10 @@
 from unittest import mock
 
 import pytest
+from boto3 import client
 from moto import mock_aws
 
+from airflow.exceptions import AirflowException, TaskDeferred
 from airflow.providers.amazon.aws.hooks.neptune import NeptuneHook
 from airflow.providers.amazon.aws.operators.neptune import (
     NeptuneStartDbClusterOperator,
@@ -100,11 +102,113 @@ def test_start_cluster_cluster_available(self, mock_waiter, mock_get_cluster_sta
         mock_waiter.assert_not_called()
         assert resp == {"db_cluster_id": CLUSTER_ID}
 
+    @mock.patch.object(NeptuneHook, "conn")
+    def test_start_cluster_deferrable(self, mock_conn):
+        operator = NeptuneStartDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=True,
+            wait_for_completion=False,
+            aws_conn_id="aws_default",
+        )
+
+        with pytest.raises(TaskDeferred):
+            operator.execute(None)
+
+    @mock.patch.object(NeptuneHook, "conn")
+    @mock.patch.object(NeptuneHook, "get_cluster_status")
+    @mock.patch.object(NeptuneHook, "get_waiter")
+    def test_start_cluster_cluster_error(self, mock_waiter, mock_get_cluster_status, mock_conn):
+        mock_get_cluster_status.return_value = "migration-failed"
+        operator = NeptuneStartDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=False,
+            wait_for_completion=True,
+            aws_conn_id="aws_default",
+        )
+
+        with pytest.raises(AirflowException):
+            operator.execute(None)
+
+    @mock.patch("airflow.providers.amazon.aws.operators.neptune.NeptuneStartDbClusterOperator.defer")
+    @mock.patch("airflow.providers.amazon.aws.operators.neptune.handle_waitable_exception")
+    @mock.patch.object(NeptuneHook, "conn")
+    def test_start_cluster_not_ready_defer(self, mock_conn, mock_wait, mock_defer):
+        err_response = {"Error": {"Code": "InvalidClusterState", "Message": "Test message"}}
+        exception = client("neptune").exceptions.ClientError(err_response, "test")
+        returned_exception = type(exception)
+
+        mock_conn.exceptions.InvalidClusterStateFault = returned_exception
+        mock_conn.start_db_cluster.side_effect = exception
+        operator = NeptuneStartDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=True,
+            wait_for_completion=False,
+            aws_conn_id="aws_default",
+        )
+
+        operator.execute(None)
+
+        mock_wait.assert_called_once_with(
+            operator=operator,
+            err="InvalidClusterState",
+        )
+        assert mock_defer.call_count == 1
+
+    @mock.patch.object(NeptuneHook, "get_waiter")
+    @mock.patch.object(NeptuneHook, "conn")
+    def test_start_cluster_instances_not_ready(self, mock_conn, mock_get_waiter):
+        err_response = {"Error": {"Code": "InvalidDBInstanceState", "Message": "Test message"}}
+        exception = client("neptune").exceptions.ClientError(err_response, "test")
+        returned_exception = type(exception)
+
+        mock_conn.exceptions.InvalidClusterStateFault = returned_exception
+        mock_conn.start_db_cluster.side_effect = exception
+
+        operator = NeptuneStartDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=False,
+            wait_for_completion=False,
+            aws_conn_id="aws_default",
+        )
+        operator.execute(None)
+        mock_get_waiter.assert_any_call("db_instance_available")
+
+    @mock.patch("airflow.providers.amazon.aws.operators.neptune.NeptuneStartDbClusterOperator.defer")
+    @mock.patch.object(NeptuneHook, "conn")
+    def test_start_cluster_instances_not_ready_defer(self, mock_conn, mock_defer):
+        """Tests both waiters are called if an instance exception is raised"""
+
+        err_response = {"Error": {"Code": "InvalidDBInstanceState", "Message": "Test message"}}
+        exception = client("neptune").exceptions.ClientError(err_response, "test")
+        returned_exception = type(exception)
+
+        mock_conn.exceptions.InvalidClusterStateFault = returned_exception
+        mock_conn.start_db_cluster.side_effect = exception
+        operator = NeptuneStartDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=True,
+            wait_for_completion=False,
+            aws_conn_id="aws_default",
+        )
+
+        operator.execute(None)
+
+        # mock_defer.assert_has_calls(calls)
+        assert mock_defer.call_count == 2
+
 
 class TestNeptuneStopClusterOperator:
     @mock.patch.object(NeptuneHook, "conn")
+    @mock.patch.object(NeptuneHook, "get_cluster_status")
     @mock.patch.object(NeptuneHook, "get_waiter")
-    def test_stop_cluster_wait_for_completion(self, mock_hook_get_waiter, mock_conn):
+    def test_stop_cluster_wait_for_completion(self, mock_hook_get_waiter, mock_get_cluster_status, mock_conn):
+        '''Test the waiter is only once when the cluster is "available"'''
+        mock_get_cluster_status.return_value = "available"
         operator = NeptuneStopDbClusterOperator(
             task_id="task_test",
             db_cluster_id=CLUSTER_ID,
@@ -118,8 +222,11 @@ def test_stop_cluster_wait_for_completion(self, mock_hook_get_waiter, mock_conn)
         assert resp == EXPECTED_RESPONSE
 
     @mock.patch.object(NeptuneHook, "conn")
+    @mock.patch.object(NeptuneHook, "get_cluster_status")
     @mock.patch.object(NeptuneHook, "get_waiter")
-    def test_stop_cluster_no_wait(self, mock_hook_get_waiter, mock_conn):
+    def test_stop_cluster_no_wait(self, mock_hook_get_waiter, mock_get_cluster_status, mock_conn):
+        mock_get_cluster_status.return_value = "available"
+
         operator = NeptuneStopDbClusterOperator(
             task_id="task_test",
             db_cluster_id=CLUSTER_ID,
@@ -150,3 +257,114 @@ def test_stop_cluster_cluster_stopped(self, mock_waiter, mock_get_cluster_status
         mock_conn.stop_db_cluster.assert_not_called()
         mock_waiter.assert_not_called()
         assert resp == {"db_cluster_id": CLUSTER_ID}
+
+    @mock.patch.object(NeptuneHook, "conn")
+    @mock.patch.object(NeptuneHook, "get_cluster_status")
+    @mock.patch.object(NeptuneHook, "get_waiter")
+    def test_stop_cluster_cluster_error(self, mock_waiter, mock_get_cluster_status, mock_conn):
+        mock_get_cluster_status.return_value = "migration-failed"
+        operator = NeptuneStopDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=False,
+            wait_for_completion=True,
+            aws_conn_id="aws_default",
+        )
+
+        with pytest.raises(AirflowException):
+            operator.execute(None)
+
+    @mock.patch.object(NeptuneHook, "conn")
+    @mock.patch.object(NeptuneHook, "get_cluster_status")
+    @mock.patch.object(NeptuneHook, "get_waiter")
+    def test_stop_cluster_not_in_available(self, mock_waiter, mock_get_cluster_status, mock_conn):
+        mock_get_cluster_status.return_value = "backing-up"
+        operator = NeptuneStopDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=False,
+            wait_for_completion=True,
+            aws_conn_id="aws_default",
+        )
+
+        operator.execute(None)
+        mock_waiter.assert_called_with("cluster_stopped")
+
+    @mock.patch.object(NeptuneStopDbClusterOperator, "defer")
+    @mock.patch.object(NeptuneHook, "conn")
+    def test_stop_cluster_not_ready_defer(self, mock_conn, mock_defer):
+        err_response = {"Error": {"Code": "InvalidClusterState", "Message": "Test message"}}
+        exception = client("neptune").exceptions.ClientError(err_response, "test")
+        returned_exception = type(exception)
+
+        mock_conn.exceptions.InvalidClusterStateFault = returned_exception
+        mock_conn.stop_db_cluster.side_effect = exception
+        operator = NeptuneStopDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=True,
+            wait_for_completion=False,
+            aws_conn_id="aws_default",
+        )
+
+        operator.execute(None)
+        assert mock_defer.call_count == 2
+
+    @mock.patch.object(NeptuneHook, "conn")
+    @mock.patch.object(NeptuneHook, "get_cluster_status")
+    @mock.patch.object(NeptuneHook, "get_waiter")
+    def test_stop_cluster_instances_not_ready(self, mock_get_waiter, mock_get_cluster_status, mock_conn):
+        err_response = {"Error": {"Code": "InvalidDBInstanceState", "Message": "Test message"}}
+        exception = client("neptune").exceptions.ClientError(err_response, "test")
+        returned_exception = type(exception)
+
+        mock_conn.exceptions.InvalidClusterStateFault = returned_exception
+        mock_conn.stop_db_cluster.side_effect = exception
+
+        operator = NeptuneStopDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=False,
+            wait_for_completion=False,
+            aws_conn_id="aws_default",
+        )
+        operator.execute(None)
+        mock_get_waiter.assert_any_call("db_instance_available")
+
+    @mock.patch("airflow.providers.amazon.aws.operators.neptune.NeptuneStopDbClusterOperator.defer")
+    @mock.patch.object(NeptuneHook, "conn")
+    def test_stop_cluster_instances_not_ready_defer(self, mock_conn, mock_defer):
+        """Tests both waiters are called if an instance exception is raised"""
+
+        err_response = {"Error": {"Code": "InvalidDBInstanceState", "Message": "Test message"}}
+        exception = client("neptune").exceptions.ClientError(err_response, "test")
+        returned_exception = type(exception)
+
+        mock_conn.exceptions.InvalidClusterStateFault = returned_exception
+        mock_conn.stop_db_cluster.side_effect = exception
+        operator = NeptuneStopDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=True,
+            wait_for_completion=False,
+            aws_conn_id="aws_default",
+        )
+
+        operator.execute(None)
+        # setting the trigger to ANY because an actual trigger doesn't seem to pass the test, even though
+        # the output (strings) are exactly the same. Suspect it may be an issue in the trigger's __eq__ function
+
+        assert mock_defer.call_count == 2
+
+    @mock.patch.object(NeptuneHook, "conn")
+    def test_stop_cluster_deferrable(self, mock_conn):
+        operator = NeptuneStopDbClusterOperator(
+            task_id="task_test",
+            db_cluster_id=CLUSTER_ID,
+            deferrable=True,
+            wait_for_completion=False,
+            aws_conn_id="aws_default",
+        )
+
+        with pytest.raises(TaskDeferred):
+            operator.execute(None)
diff --git a/tests/providers/amazon/aws/triggers/test_neptune.py b/tests/providers/amazon/aws/triggers/test_neptune.py
--- a/tests/providers/amazon/aws/triggers/test_neptune.py
+++ b/tests/providers/amazon/aws/triggers/test_neptune.py
@@ -20,9 +20,12 @@
 from unittest.mock import AsyncMock
 
 import pytest
+from botocore.exceptions import WaiterError
 
+from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.triggers.neptune import (
     NeptuneClusterAvailableTrigger,
+    NeptuneClusterInstancesAvailableTrigger,
     NeptuneClusterStoppedTrigger,
 )
 from airflow.triggers.base import TriggerEvent
@@ -80,3 +83,47 @@ async def test_run_success(self, mock_async_conn, mock_get_waiter):
 
         assert resp == TriggerEvent({"status": "success", "db_cluster_id": CLUSTER_ID})
         assert mock_get_waiter().wait.call_count == 1
+
+
+class TestNeptuneClusterInstancesAvailableTrigger:
+    def test_serialization(self):
+        """
+        Asserts that the TaskStateTrigger correctly serializes its arguments
+        and classpath.
+        """
+        trigger = NeptuneClusterInstancesAvailableTrigger(db_cluster_id=CLUSTER_ID)
+        classpath, kwargs = trigger.serialize()
+        assert (
+            classpath
+            == "airflow.providers.amazon.aws.triggers.neptune.NeptuneClusterInstancesAvailableTrigger"
+        )
+        assert "db_cluster_id" in kwargs
+        assert kwargs["db_cluster_id"] == CLUSTER_ID
+
+    @pytest.mark.asyncio
+    @mock.patch("airflow.providers.amazon.aws.hooks.neptune.NeptuneHook.get_waiter")
+    @mock.patch("airflow.providers.amazon.aws.hooks.neptune.NeptuneHook.async_conn")
+    async def test_run_success(self, mock_async_conn, mock_get_waiter):
+        mock_async_conn.__aenter__.return_value = "available"
+        mock_get_waiter().wait = AsyncMock()
+        trigger = NeptuneClusterInstancesAvailableTrigger(db_cluster_id=CLUSTER_ID)
+        generator = trigger.run()
+        resp = await generator.asend(None)
+
+        assert resp == TriggerEvent({"status": "success", "db_cluster_id": CLUSTER_ID})
+        assert mock_get_waiter().wait.call_count == 1
+
+    @pytest.mark.asyncio
+    @mock.patch("airflow.providers.amazon.aws.hooks.neptune.NeptuneHook.async_conn")
+    async def test_run_fail(self, mock_async_conn):
+        a_mock = mock.MagicMock()
+        mock_async_conn.__aenter__.return_value = a_mock
+        wait_mock = AsyncMock()
+        wait_mock.side_effect = WaiterError("name", "reason", {"test": [{"lastStatus": "my_status"}]})
+        a_mock.get_waiter().wait = wait_mock
+        trigger = NeptuneClusterInstancesAvailableTrigger(
+            db_cluster_id=CLUSTER_ID, waiter_delay=1, waiter_max_attempts=2
+        )
+
+        with pytest.raises(AirflowException):
+            await trigger.run().asend(None)