Repository: tensorflow/tfx

Issue:
Workshop tutorial bug: unexpected keyword argument 'components'
The commit 7541df5 seems to break the [TFX Developer Tutorial](https://www.tensorflow.org/tfx/tutorials/tfx/workshop) in step 2, when executing:

```shell
# Open a new terminal window, and in that window ...
source ~/tfx-env/bin/activate
airflow webserver -p 8080
```

It outputs the error message:

```
[2019-04-25 18:57:19,644] {__init__.py:416} ERROR - Failed to import: /home/j3soon/airflow/dags/taxi_pipeline_solution.py
Traceback (most recent call last):
  File "/home/j3soon/tfx-env/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 413, in process_file
    m = imp.load_source(mod_name, filepath)
  File "/home/j3soon/airflow/dags/taxi_pipeline_solution.py", line 147, in <module>
    taxi_pipeline = AirflowDAGRunner(_airflow_config).run(_create_pipeline())
  File "/home/j3soon/tfx-env/local/lib/python2.7/site-packages/tfx/orchestration/airflow/airflow_runner.py", line 45, in run
    airflow_dag = airflow_pipeline.AirflowPipeline(**self._config)
TypeError: __init__() got an unexpected keyword argument 'components'
Running the Gunicorn Server with:
Workers: 4 sync
Host: 0.0.0.0:8080
Timeout: 120
Logfiles: - -
```

My partner following the tutorial last week and it works perfectly. However, when I'm trying to follow the tutorial again on the same machine yesterday, the error occurs.

After replacing the file `/home/j3soon/airflow/dags/taxi_pipeline_solution.py` with the old file, the error is resolved.


Discussion:
Thanks for reporting this. I tried at my end and found the same error with the latest tutorial version. We will look into it.
This error is caused by [this](https://github.com/tensorflow/tfx/commit/7541df5f3e4dd9b2237b3105b0784f640249efc4) commit which added an extra argument in the `__init__()` function in `Pipeline`. However, the tutorial still pins to our first release version which is not compatible. We will have a PR to fix this soon.

Resolving patch:
diff --git a/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_kubeflow.py b/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_kubeflow.py
--- a/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_kubeflow.py
+++ b/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_kubeflow.py
@@ -27,8 +27,8 @@
 from tfx.components.statistics_gen.component import StatisticsGen
 from tfx.components.trainer.component import Trainer
 from tfx.components.transform.component import Transform
-from tfx.orchestration import pipeline
 from tfx.orchestration.kubeflow.runner import KubeflowRunner
+from tfx.orchestration.pipeline import PipelineDecorator
 from tfx.proto import evaluator_pb2
 from tfx.proto import pusher_pb2
 from tfx.proto import trainer_pb2
@@ -87,6 +87,22 @@
 _query_sample_rate = 0.001  # Generate a 0.1% random sample.
 
 
+# TODO(zhitaoli): Remove PipelineDecorator after 0.13.0.
+@PipelineDecorator(
+    pipeline_name='chicago_taxi_pipeline_kubeflow',
+    log_root='/var/tmp/tfx/logs',
+    pipeline_root=_pipeline_root,
+    additional_pipeline_args={
+        'beam_pipeline_args': [
+            '--runner=DataflowRunner',
+            '--experiments=shuffle_mode=auto',
+            '--project=' + _project_id,
+            '--temp_location=' + os.path.join(_output_bucket, 'tmp'),
+            '--region=' + _gcp_region,
+        ],
+        # Optional args:
+        # 'tfx_image': custom docker image to use for components.
+    })
 def _create_pipeline():
   """Implements the chicago taxi pipeline with TFX."""
 
@@ -165,33 +181,10 @@ def _create_pipeline():
           filesystem=pusher_pb2.PushDestination.Filesystem(
               base_directory=_serving_model_dir)))
 
-  return pipeline.Pipeline(
-      pipeline_name='chicago_taxi_pipeline_kubeflow',
-      pipeline_root=_pipeline_root,
-      components=[
-          example_gen,
-          statistics_gen,
-          infer_schema,
-          validate_stats,
-          transform,
-          trainer,
-          model_analyzer,
-          model_validator,
-          pusher,
-      ],
-      log_root='/var/tmp/tfx/logs',
-      additional_pipeline_args={
-          'beam_pipeline_args': [
-              '--runner=DataflowRunner',
-              '--experiments=shuffle_mode=auto',
-              '--project=' + _project_id,
-              '--temp_location=' + os.path.join(_output_bucket, 'tmp'),
-              '--region=' + _gcp_region,
-          ],
-          # Optional args:
-          # 'tfx_image': custom docker image to use for components.
-      },
-  )
-
-
-taxi_pipeline = KubeflowRunner().run(_create_pipeline())
+  return [
+      example_gen, statistics_gen, infer_schema, validate_stats, transform,
+      trainer, model_analyzer, model_validator, pusher
+  ]
+
+
+pipeline = KubeflowRunner().run(_create_pipeline())
diff --git a/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_simple.py b/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_simple.py
--- a/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_simple.py
+++ b/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_simple.py
@@ -29,8 +29,8 @@
 from tfx.components.statistics_gen.component import StatisticsGen
 from tfx.components.trainer.component import Trainer
 from tfx.components.transform.component import Transform
-from tfx.orchestration import pipeline
 from tfx.orchestration.airflow.airflow_runner import AirflowDAGRunner
+from tfx.orchestration.pipeline import PipelineDecorator
 from tfx.proto import evaluator_pb2
 from tfx.proto import pusher_pb2
 from tfx.proto import trainer_pb2
@@ -70,6 +70,13 @@
 
 
 # TODO(b/124066911): Centralize tfx related config into one place.
+# TODO(zhitaoli): Remove PipelineDecorator after 0.13.0.
+@PipelineDecorator(
+    pipeline_name='chicago_taxi_simple',
+    enable_cache=True,
+    metadata_db_root=_metadata_db_root,
+    additional_pipeline_args={'logger_args': logger_overrides},
+    pipeline_root=_pipeline_root)
 def _create_pipeline():
   """Implements the chicago taxi pipeline with TFX."""
   examples = csv_input(_data_root)
@@ -124,24 +131,10 @@ def _create_pipeline():
           filesystem=pusher_pb2.PushDestination.Filesystem(
               base_directory=_serving_model_dir)))
 
-  return pipeline.Pipeline(
-      pipeline_name='chicago_taxi_simple',
-      pipeline_root=_pipeline_root,
-      components=[
-          example_gen,
-          statistics_gen,
-          infer_schema,
-          validate_stats,
-          transform,
-          trainer,
-          model_analyzer,
-          model_validator,
-          pusher,
-      ],
-      enable_cache=True,
-      metadata_db_root=_metadata_db_root,
-      additional_pipeline_args={'logger_args': logger_overrides},
-  )
-
-
-taxi_pipeline = AirflowDAGRunner(_airflow_config).run(_create_pipeline())
+  return [
+      example_gen, statistics_gen, infer_schema, validate_stats, transform,
+      trainer, model_analyzer, model_validator, pusher
+  ]
+
+
+pipeline = AirflowDAGRunner(_airflow_config).run(_create_pipeline())


Test patch: