Repository: apache/airflow

Issue:
Test that dataset not updated when task skipped
the AIP specifies that when a task is skipped, that we don’t mark the dataset as “updated”. we should simply add a test that verifies that this is what happens (and make changes if necessary) 

@blag, i tried to make this an issue so i could assign to you but can't. anyway, can reference in PR with `closes`


Discussion:

Resolving patch:
diff --git a/airflow/example_dags/example_datasets.py b/airflow/example_dags/example_datasets.py
--- a/airflow/example_dags/example_datasets.py
+++ b/airflow/example_dags/example_datasets.py
@@ -34,16 +34,24 @@
 
 Dags 5 and 6 should not run because they depend on datasets that never get updated.
 
+DAG dag7 should skip its only task and never trigger dag8
+
+DAG dag9 should fail its only task and never trigger dag10
+
 """
 from datetime import datetime
 
+from airflow.exceptions import AirflowFailException, AirflowSkipException
 from airflow.models import DAG, Dataset
 from airflow.operators.bash import BashOperator
+from airflow.operators.python import PythonOperator
 
 # [START dataset_def]
 dag1_dataset = Dataset('s3://dag1/output_1.txt', extra={'hi': 'bye'})
 # [END dataset_def]
 dag2_dataset = Dataset('s3://dag2/output_1.txt', extra={'hi': 'bye'})
+dag7_dataset = Dataset('s3://dag7/output_1.txt', extra={'hi': 'bye'})
+dag9_dataset = Dataset('s3://dag9/output_1.txt', extra={'hi': 'bye'})
 
 dag1 = DAG(
     dag_id='dag1',
@@ -131,3 +139,65 @@
         outlets=[Dataset('s3://unrelated_task/dataset_other_unknown.txt')],
         bash_command="sleep 5",
     )
+
+
+def raise_skip_exc():
+    raise AirflowSkipException
+
+
+dag7 = DAG(
+    dag_id='dag7',
+    catchup=False,
+    start_date=datetime(2020, 1, 1),
+    schedule_interval='@daily',
+    tags=['upstream-skipping'],
+)
+PythonOperator(
+    task_id='skip_task',
+    outlets=[dag7_dataset],
+    python_callable=raise_skip_exc,
+    dag=dag7,
+)
+
+with DAG(
+    dag_id='dag8',
+    catchup=False,
+    start_date=datetime(2020, 1, 1),
+    schedule_on=[dag7_dataset],
+    tags=['downstream-skipped'],
+) as dag8:
+    BashOperator(
+        task_id='dag8_task',
+        bash_command="sleep 5",
+    )
+
+
+def raise_assertionerror():
+    raise AirflowFailException
+
+
+dag9 = DAG(
+    dag_id='dag9',
+    catchup=False,
+    start_date=datetime(2020, 1, 1),
+    schedule_interval='@daily',
+    tags=['upstream-skipping'],
+)
+PythonOperator(
+    task_id='fail_task',
+    outlets=[dag9_dataset],
+    python_callable=raise_assertionerror,
+    dag=dag9,
+)
+
+with DAG(
+    dag_id='dag10',
+    catchup=False,
+    start_date=datetime(2020, 1, 1),
+    schedule_on=[dag9_dataset],
+    tags=['downstream-failed'],
+) as dag10:
+    BashOperator(
+        task_id='dag10_task',
+        bash_command="sleep 5",
+    )
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -1518,7 +1518,8 @@ def _run_raw_task(
         if not test_mode:
             session.add(Log(self.state, self))
             session.merge(self)
-            self._create_dataset_dag_run_queue_records(session=session)
+            if self.state == TaskInstanceState.SUCCESS:
+                self._create_dataset_dag_run_queue_records(session=session)
             session.commit()
 
     def _create_dataset_dag_run_queue_records(self, *, session: Session) -> None:


Test patch:
diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py
--- a/tests/models/test_taskinstance.py
+++ b/tests/models/test_taskinstance.py
@@ -1535,7 +1535,7 @@ def test_outlet_datasets(self, create_task_instance):
         session.commit()
         ti._run_raw_task()
         ti.refresh_from_db()
-        assert ti.state == State.SUCCESS
+        assert ti.state == TaskInstanceState.SUCCESS
 
         # check that one queue record created for each dag that depends on dataset 1
         assert session.query(DatasetDagRunQueue.target_dag_id).filter(
@@ -1555,6 +1555,70 @@ def test_outlet_datasets(self, create_task_instance):
             .count()
         ) == 1
 
+        # Clean up after ourselves
+        db.clear_db_datasets()
+
+    def test_outlet_datasets_failed(self, create_task_instance):
+        """
+        Verify that when we have an outlet dataset on a task, and the task
+        failed, a DatasetDagRunQueue is not logged, and a DatasetEvent is
+        not generated
+        """
+        from airflow.example_dags import example_datasets
+        from airflow.example_dags.example_datasets import dag9
+
+        session = settings.Session()
+        dagbag = DagBag(dag_folder=example_datasets.__file__)
+        dagbag.collect_dags(only_if_updated=False, safe_mode=False)
+        dagbag.sync_to_db(session=session)
+        run_id = str(uuid4())
+        dr = DagRun(dag9.dag_id, run_id=run_id, run_type='anything')
+        session.merge(dr)
+        task = dag9.get_task('fail_task')
+        ti = TaskInstance(task, run_id=run_id)
+        session.merge(ti)
+        session.commit()
+        with pytest.raises(AirflowFailException):
+            ti._run_raw_task()
+        ti.refresh_from_db()
+        assert ti.state == TaskInstanceState.FAILED
+
+        # check that no dagruns were queued
+        assert session.query(DatasetDagRunQueue).count() == 0
+
+        # check that no dataset events were generated
+        assert session.query(DatasetEvent).count() == 0
+
+    def test_outlet_datasets_skipped(self, create_task_instance):
+        """
+        Verify that when we have an outlet dataset on a task, and the task
+        is skipped, a DatasetDagRunQueue is not logged, and a DatasetEvent is
+        not generated
+        """
+        from airflow.example_dags import example_datasets
+        from airflow.example_dags.example_datasets import dag7
+
+        session = settings.Session()
+        dagbag = DagBag(dag_folder=example_datasets.__file__)
+        dagbag.collect_dags(only_if_updated=False, safe_mode=False)
+        dagbag.sync_to_db(session=session)
+        run_id = str(uuid4())
+        dr = DagRun(dag7.dag_id, run_id=run_id, run_type='anything')
+        session.merge(dr)
+        task = dag7.get_task('skip_task')
+        ti = TaskInstance(task, run_id=run_id)
+        session.merge(ti)
+        session.commit()
+        ti._run_raw_task()
+        ti.refresh_from_db()
+        assert ti.state == TaskInstanceState.SKIPPED
+
+        # check that no dagruns were queued
+        assert session.query(DatasetDagRunQueue).count() == 0
+
+        # check that no dataset events were generated
+        assert session.query(DatasetEvent).count() == 0
+
     @staticmethod
     def _test_previous_dates_setup(
         schedule_interval: Union[str, datetime.timedelta, None],