Repository: aws/sagemaker-python-sdk

Issue:
Running multiple Transform jobs with same Transformer uses same output_path for all transform jobs
Please fill out the form below.

### System Information
- **Framework (e.g. TensorFlow) / Algorithm (e.g. KMeans)**: TensorFlow
- **Framework Version**: 1.12.0
- **Python Version**: 3
- **CPU or GPU**: CPU
- **Python SDK Version**: sagemaker==1.16.3
- **Are you using a custom image**: No.

### Describe the problem

If I run more than one Transform Job using a Transformer, the outputs are written to the same S3 output path, even though I'm running different transform jobs. So outputs can collide and be overwritten or I'll end up with outputs from multiple jobs in the same S3 output path

### Minimal repro / logs
I'm running tensorflow_batch_transform_mnist.ipynb.

You can reproduce this by doing:
transformer.transform([your batch s3 input path], content_type='text/csv')
print(transformer.output_path)

transformer.transform([some other batch s3 input path], content_type='text/csv')
print(transformer.output_path)

And the transformer.output_path will be the same (but the jobs are different)


Discussion:
So current behavior is to pass to the platform s3 location provided by the user or create a job_id based folder inside default s3 bucket (similar to what SageMaker platform does for training output):
https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/transformer.py#L111-L112

Moving this issue to the platform team to answer why the collision use case is not addressed at the platform level.

The issue is:

If I run one job with default generated name "sagemaker-tensorflow-1", that gets written to "s3:/...../sagemaker-tensorflow-1/".

If I run another job with the same transformer with default generated name "sagemaker-tensorflow-2", that gets written to "s3:/...../sagemaker-tensorflow-1", instead of "s3:/...../sagemaker-tensorflow-2", which means Python SDK tells the platform to write output from both jobs to the same output location. I don't think that's expected (or good for users), regardless of what the platform does -- wouldn't it make more sense to default to telling the platform to write to a prefix with the job name?
I'd like to add that I think this is a little weird from an API perspective, since the [`CreateTransformJob`](https://docs.aws.amazon.com/sagemaker/latest/dg/API_CreateTransformJob.html) that the `Transformer.transform` method presumably would correspond to takes a `TransformOutput` as an argument. Is there a specific reason that this has to be provided on the creation of the Transformer object?

Resolving patch:
diff --git a/src/sagemaker/transformer.py b/src/sagemaker/transformer.py
--- a/src/sagemaker/transformer.py
+++ b/src/sagemaker/transformer.py
@@ -90,6 +90,7 @@ def __init__(
         self.base_transform_job_name = base_transform_job_name
         self._current_job_name = None
         self.latest_transform_job = None
+        self._reset_output_path = False
 
         self.sagemaker_session = sagemaker_session or Session()
 
@@ -146,10 +147,11 @@ def transform(
 
             self._current_job_name = name_from_base(base_name)
 
-        if self.output_path is None:
+        if self.output_path is None or self._reset_output_path is True:
             self.output_path = "s3://{}/{}".format(
                 self.sagemaker_session.default_bucket(), self._current_job_name
             )
+            self._reset_output_path = True
 
         self.latest_transform_job = _TransformJob.start_new(
             self,


Test patch:
diff --git a/tests/integ/test_transformer.py b/tests/integ/test_transformer.py
--- a/tests/integ/test_transformer.py
+++ b/tests/integ/test_transformer.py
@@ -301,6 +301,53 @@ def test_transform_byo_estimator(sagemaker_session):
         assert tags == model_tags
 
 
+def test_single_transformer_multiple_jobs(sagemaker_session, mxnet_full_version):
+    data_path = os.path.join(DATA_DIR, "mxnet_mnist")
+    script_path = os.path.join(data_path, "mnist.py")
+
+    mx = MXNet(
+        entry_point=script_path,
+        role="SageMakerRole",
+        train_instance_count=1,
+        train_instance_type="ml.c4.xlarge",
+        sagemaker_session=sagemaker_session,
+        framework_version=mxnet_full_version,
+    )
+
+    train_input = mx.sagemaker_session.upload_data(
+        path=os.path.join(data_path, "train"), key_prefix="integ-test-data/mxnet_mnist/train"
+    )
+    test_input = mx.sagemaker_session.upload_data(
+        path=os.path.join(data_path, "test"), key_prefix="integ-test-data/mxnet_mnist/test"
+    )
+    job_name = unique_name_from_base("test-mxnet-transform")
+
+    with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
+        mx.fit({"train": train_input, "test": test_input}, job_name=job_name)
+
+    transform_input_path = os.path.join(data_path, "transform", "data.csv")
+    transform_input_key_prefix = "integ-test-data/mxnet_mnist/transform"
+    transform_input = mx.sagemaker_session.upload_data(
+        path=transform_input_path, key_prefix=transform_input_key_prefix
+    )
+
+    transformer = mx.transformer(1, "ml.m4.xlarge")
+
+    job_name = unique_name_from_base("test-mxnet-transform")
+    transformer.transform(transform_input, content_type="text/csv", job_name=job_name)
+    with timeout_and_delete_model_with_transformer(
+        transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
+    ):
+        assert transformer.output_path == "s3://{}/{}".format(
+            sagemaker_session.default_bucket(), job_name
+        )
+        job_name = unique_name_from_base("test-mxnet-transform")
+        transformer.transform(transform_input, content_type="text/csv", job_name=job_name)
+        assert transformer.output_path == "s3://{}/{}".format(
+            sagemaker_session.default_bucket(), job_name
+        )
+
+
 def _create_transformer_and_transform_job(
     estimator,
     transform_input,
diff --git a/tests/unit/test_transformer.py b/tests/unit/test_transformer.py
--- a/tests/unit/test_transformer.py
+++ b/tests/unit/test_transformer.py
@@ -437,3 +437,15 @@ def test_transform_job_wait(sagemaker_session):
     job.wait()
 
     assert sagemaker_session.wait_for_transform_job.called_once
+
+
+@patch("sagemaker.transformer._TransformJob.start_new")
+def test_restart_output_path(start_new_job, transformer, sagemaker_session):
+    transformer.output_path = None
+    sagemaker_session.default_bucket.return_value = S3_BUCKET
+
+    transformer.transform(DATA, job_name="job-1")
+    assert transformer.output_path == "s3://{}/{}".format(S3_BUCKET, "job-1")
+
+    transformer.transform(DATA, job_name="job-2")
+    assert transformer.output_path == "s3://{}/{}".format(S3_BUCKET, "job-2")