Repository: tensorflow/tfx

Issue:
AI Platform Pusher fails on 2nd run
I am imitating `_ai_platform_serving_args` of [taxi_pipeline_kubeflow.py](https://github.com/tensorflow/tfx/blob/46bb4f975c36ea1defde4b3c33553e088b3dc5b8/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_kubeflow.py#L85)

When I execute the Pusher for 2nd time, it fails on the following error, as opposed to creating a new version under the same model name.

```
Traceback (most recent call last):
  File "/opt/venv/lib/python3.6/site-packages/tfx/extensions/google_cloud_ai_platform/runner.py", line 176, in deploy_model_for_cmle_serving
    api.projects().models().create(body=body, parent=parent).execute()
  File "/opt/venv/lib/python3.6/site-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
    return wrapped(*args, **kwargs)
  File "/opt/venv/lib/python3.6/site-packages/googleapiclient/http.py", line 856, in execute
    raise HttpError(resp, content, uri=self.uri)
googleapiclient.errors.HttpError: <HttpError 409 when requesting 
https://ml.googleapis.com/v1/projects/xxxxxxxx/models?alt=json
 returned "Field: model.name Error: A model with the same name already exists.". Details: "[{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'model.name', 'description': 'A model with the same name already exists.'}]}]">
```

 I would expect that a new version with `v{{unix timestamp}}` to be created under the same model name.


Discussion:

Resolving patch:
diff --git a/tfx/extensions/google_cloud_ai_platform/runner.py b/tfx/extensions/google_cloud_ai_platform/runner.py
--- a/tfx/extensions/google_cloud_ai_platform/runner.py
+++ b/tfx/extensions/google_cloud_ai_platform/runner.py
@@ -177,7 +177,7 @@ def deploy_model_for_cmle_serving(serving_path: Text, model_version: Text,
   except errors.HttpError as e:
     # If the error is to create an already existing model, it's ok to ignore.
     # TODO(b/135211463): Remove the disable once the pytype bug is fixed.
-    if e.resp.status == '409':  # pytype: disable=attribute-error
+    if e.resp.status == 409:  # pytype: disable=attribute-error
       tf.logging.warn('Model {} already exists'.format(model_name))
     else:
       raise RuntimeError('CMLE Push failed: {}'.format(e))


Test patch:
diff --git a/tfx/orchestration/kubeflow/kubeflow_gcp_integration_test.py b/tfx/orchestration/kubeflow/kubeflow_gcp_integration_test.py
--- a/tfx/orchestration/kubeflow/kubeflow_gcp_integration_test.py
+++ b/tfx/orchestration/kubeflow/kubeflow_gcp_integration_test.py
@@ -147,8 +147,10 @@ def _delete_ai_platform_model(self, model_name):
       tf.logging.info('Model %s has versions %s' % (model_name, versions))
 
       # First line of the output is the header: [NAME] [DEPLOYMENT_URI] [STATE]
+      # Second line is the 'default' version, which needs to be deleted last,
+      # as such, need to flip the order of result lines.
       for version in versions.stdout.decode('utf-8').strip('\n').split(
-          '\n')[1:]:
+          '\n')[1:][::-1]:
         version = version.split()[0]
         tf.logging.info('Deleting version %s of model %s' %
                         (version, model_name))
@@ -204,10 +206,15 @@ def setUp(self):
     self._test_transform_graph = [transform_graph]
 
     # Model artifact for testing.
-    model = standard_artifacts.Model()
-    model.uri = os.path.join(self._intermediate_data_root,
-                             'trainer/output/test-pipeline/')
-    self._test_model = [model]
+    model_1 = standard_artifacts.Model()
+    model_1.uri = os.path.join(self._intermediate_data_root,
+                               'trainer/output/test-pipeline/1/')
+    self._test_model_1 = [model_1]
+
+    model_2 = standard_artifacts.Model()
+    model_2.uri = os.path.join(self._intermediate_data_root,
+                               'trainer/output/test-pipeline/2/')
+    self._test_model_2 = [model_2]
 
     # ModelBlessing artifact for testing.
     model_blessing = standard_artifacts.ModelBlessing()
@@ -257,7 +264,7 @@ def testEvaluatorOnDataflowRunner(self):
             examples=self._input_artifacts(pipeline_name,
                                            self._test_raw_examples),
             model_exports=self._input_artifacts(pipeline_name,
-                                                self._test_model),
+                                                self._test_model_1),
             feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
                 evaluator_pb2.SingleSlicingSpec(
                     column_for_slicing=['trip_start_hour'])
@@ -273,7 +280,7 @@ def testModelValidatorOnDataflowRunner(self):
         ModelValidator(
             examples=self._input_artifacts(pipeline_name,
                                            self._test_raw_examples),
-            model=self._input_artifacts(pipeline_name, self._test_model))
+            model=self._input_artifacts(pipeline_name, self._test_model_1))
     ])
     self._compile_and_run_pipeline(pipeline)
 
@@ -313,19 +320,27 @@ def testAIPlatformPusherPipeline(self):
     pipeline_name = 'kubeflow-aip-pusher-test-{}'.format(self._random_id())
     # AI Platform does not accept '-' in the model name.
     model_name = ('%s_model' % pipeline_name).replace('-', '_')
+
+    def _pusher(model, name):
+      return Pusher(
+          custom_executor_spec=executor_spec.ExecutorClassSpec(
+              ai_platform_pusher_executor.Executor),
+          model=self._input_artifacts(pipeline_name, model),
+          model_blessing=self._input_artifacts(pipeline_name,
+                                               self._test_model_blessing),
+          custom_config={
+              'ai_platform_serving_args': {
+                  'model_name': model_name,
+                  'project_id': self._gcp_project_id,
+              }
+          },
+          instance_name=name,
+      )
+
     pipeline = self._create_pipeline(pipeline_name, [
-        Pusher(
-            custom_executor_spec=executor_spec.ExecutorClassSpec(
-                ai_platform_pusher_executor.Executor),
-            model=self._input_artifacts(pipeline_name, self._test_model),
-            model_blessing=self._input_artifacts(pipeline_name,
-                                                 self._test_model_blessing),
-            custom_config={
-                'ai_platform_serving_args': {
-                    'model_name': model_name,
-                    'project_id': self._gcp_project_id,
-                }
-            })
+        # Test creation of multiple versions under the same model_name.
+        _pusher(self._test_model_1, 'model_1'),
+        _pusher(self._test_model_2, 'model_2'),
     ])
     self.addCleanup(self._delete_ai_platform_model, model_name)
     self._compile_and_run_pipeline(pipeline)