Repository: kubeflow/pipelines

Issue:
tfx-oss sample test would fail when retry the test
When I try '/test kubeflow-pipeline-sample-test' for flakiness tests, the parameterized_tfx_oss would always fail due to the fact that they are writing to the same directory.
See: https://prow.k8s.io/view/gcs/kubernetes-jenkins/pr-logs/pull/kubeflow_pipelines/2349/kubeflow-pipeline-sample-test/1182166563743076352


Discussion:
Right... That's similar to the problem of model pushing we observed before. Because the GCS do not update if no new commit is detected, it will fail.

Let me fix it shortly. A quick solution is make the timestamp part of the pipeline root path.

Resolving patch:
diff --git a/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.py
--- a/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.py
+++ b/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.py
@@ -53,9 +53,7 @@
     value='gs://ml-pipeline-playground/tfx_taxi_simple/data')
 
 # Path of pipeline root, should be a GCS path.
-_pipeline_root_param = dsl.PipelineParam(
-    name='pipeline-root',
-    value=os.path.join('gs://your-bucket', 'tfx_taxi_simple'))
+pipeline_root = os.path.join('gs://your-bucket', 'tfx_taxi_simple')
 
 def _create_test_pipeline(pipeline_root: Text, csv_input_location: Text,
     taxi_module_file: Text, enable_cache: bool):
@@ -99,12 +97,18 @@ def _create_test_pipeline(pipeline_root: Text, csv_input_location: Text,
       ]))
   model_validator = ModelValidator(
       examples=example_gen.outputs.examples, model=trainer.outputs.output)
+
+  # Hack: ensuring push_destination can be correctly parameterized and interpreted.
+  # pipeline root will be specified as a dsl.PipelineParam with the name
+  # pipeline-root, see:
+  # https://github.com/tensorflow/tfx/blob/1c670e92143c7856f67a866f721b8a9368ede385/tfx/orchestration/kubeflow/kubeflow_dag_runner.py#L226
+  _pipeline_root_param = dsl.PipelineParam(name='pipeline-root')
   pusher = Pusher(
       model_export=trainer.outputs.output,
       model_blessing=model_validator.outputs.blessing,
       push_destination=pusher_pb2.PushDestination(
           filesystem=pusher_pb2.PushDestination.Filesystem(
-              base_directory=os.path.join(pipeline_root, 'model_serving'))))
+              base_directory=os.path.join(str(_pipeline_root_param), 'model_serving'))))
 
   return pipeline.Pipeline(
       pipeline_name='parameterized_tfx_oss',
@@ -130,9 +134,8 @@ def _get_kubeflow_metadata_config() -> kubeflow_pb2.KubeflowMetadataConfig:
 if __name__ == '__main__':
 
   enable_cache = True
-
   pipeline = _create_test_pipeline(
-      str(_pipeline_root_param),
+      pipeline_root,
       str(_data_root_param),
       str(_taxi_module_file_param),
       enable_cache=enable_cache)


Test patch:
diff --git a/test/sample-test/run_sample_test.py b/test/sample-test/run_sample_test.py
--- a/test/sample-test/run_sample_test.py
+++ b/test/sample-test/run_sample_test.py
@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import kfp
 import os
 import tarfile
+import time
 import utils
 import yamale
 import yaml
@@ -111,7 +113,9 @@ def run(self):
 
     # TODO(numerology): Special treatment for TFX::OSS sample
     if self._testname == 'parameterized_tfx_oss':
-      self._test_args['pipeline-root'] = os.path.join(self._test_args['output'], 'tfx_taxi_simple')
+      self._test_args['pipeline-root'] = os.path.join(
+          self._test_args['output'],
+          'tfx_taxi_simple_' + kfp.dsl.RUN_ID_PLACEHOLDER)
       del self._test_args['output']
 
     # Submit for pipeline running.