Repository: cloud-custodian/cloud-custodian

Issue:
GCP- dataflow deprecated view and now doesn't return complete information
Looks like gcp deprecated the dataflow list ability to get all dataflow job information. It only returns `JOB_VIEW_SUMMARY `

https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.jobs/list

In order to get the required information a get call on every resource will be required. Should this be done on the resource level as an augmentation or in an additional filter since the jobs could be filtered on state in a value filter before the extra api calls. 


Discussion:
not quite sure i'm understanding the delta on the api. re the question, it would depend on if the summary metadata is useful by itself to express a policy against, if it is then it would be appropriate as a filter with annotation for the extended data, if its not then doing the augment works.
the `jobs.list` api now only returns `JOB_VIEW_SUMMARY` which returns Project ID, Job ID, job name, job type, job status, start/end time, and Cloud SDK version details, whereas the the` jobs.get` returns the full response (below) . Since so much information is missing I think it makes sense to augment.

```
{
  "id": string,
  "projectId": string,
  "name": string,
  "type": enum (JobType),
  "environment": {
    object (Environment)
  },
  "steps": [
    {
      object (Step)
    }
  ],
  "stepsLocation": string,
  "currentState": enum (JobState),
  "currentStateTime": string,
  "requestedState": enum (JobState),
  "executionInfo": {
    object (JobExecutionInfo)
  },
  "createTime": string,
  "replaceJobId": string,
  "transformNameMapping": {
    string: string,
    ...
  },
  "clientRequestId": string,
  "replacedByJobId": string,
  "tempFiles": [
    string
  ],
  "labels": {
    string: string,
    ...
  },
  "location": string,
  "pipelineDescription": {
    object (PipelineDescription)
  },
  "stageStates": [
    {
      object (ExecutionStageState)
    }
  ],
  "jobMetadata": {
    object (JobMetadata)
  },
  "startTime": string,
  "createdFromSnapshotId": string,
  "satisfiesPzs": boolean
}
```




Sounds good we should probably do a default server side filter for active
as well to limit the query volume.


On Mon, Apr 26, 2021 at 9:32 AM Austen ***@***.***> wrote:

> the jobs.list api now only returns JOB_VIEW_SUMMARY which returns Project
> ID, Job ID, job name, job type, job status, start/end time, and Cloud SDK
> version details, whereas the the jobs.get returns the full response
> (below) . Since so much information is missing I think it makes sense to
> augment.
>
> {
>   "id": string,
>   "projectId": string,
>   "name": string,
>   "type": enum (JobType),
>   "environment": {
>     object (Environment)
>   },
>   "steps": [
>     {
>       object (Step)
>     }
>   ],
>   "stepsLocation": string,
>   "currentState": enum (JobState),
>   "currentStateTime": string,
>   "requestedState": enum (JobState),
>   "executionInfo": {
>     object (JobExecutionInfo)
>   },
>   "createTime": string,
>   "replaceJobId": string,
>   "transformNameMapping": {
>     string: string,
>     ...
>   },
>   "clientRequestId": string,
>   "replacedByJobId": string,
>   "tempFiles": [
>     string
>   ],
>   "labels": {
>     string: string,
>     ...
>   },
>   "location": string,
>   "pipelineDescription": {
>     object (PipelineDescription)
>   },
>   "stageStates": [
>     {
>       object (ExecutionStageState)
>     }
>   ],
>   "jobMetadata": {
>     object (JobMetadata)
>   },
>   "startTime": string,
>   "createdFromSnapshotId": string,
>   "satisfiesPzs": boolean
> }
>
> —
> You are receiving this because you commented.
>
>
> Reply to this email directly, view it on GitHub
> <https://github.com/cloud-custodian/cloud-custodian/issues/6600#issuecomment-826838455>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AAAFJEU7OMMT3HQOTCWL53LTKVTOXANCNFSM42I76OQA>
> .
>


Resolving patch:
diff --git a/tools/c7n_gcp/c7n_gcp/resources/dataflow.py b/tools/c7n_gcp/c7n_gcp/resources/dataflow.py
--- a/tools/c7n_gcp/c7n_gcp/resources/dataflow.py
+++ b/tools/c7n_gcp/c7n_gcp/resources/dataflow.py
@@ -1,6 +1,7 @@
 # Copyright The Cloud Custodian Authors.
 # SPDX-License-Identifier: Apache-2.0
 import jmespath
+from googleapiclient.errors import HttpError
 
 from c7n_gcp.provider import resources
 from c7n_gcp.query import QueryResourceManager, TypeInfo
@@ -31,3 +32,27 @@ def get(client, event):
                     'jobId': jmespath.search('protoPayload.request.job_id', event)
                 }
             )
+
+    def resources(self, query=None):
+        query_filter = 'ACTIVE'
+        if self.data.get('query'):
+            query_filter = self.data['query'][0].get('filter', 'ACTIVE')
+
+        return super(DataflowJob, self).resources(query={'filter': query_filter})
+
+    def augment(self, resources):
+        client = self.get_client()
+        results = []
+        for r in resources:
+            ref = {
+                'jobId': r['id'],
+                'projectId': r['projectId'],
+                'view': 'JOB_VIEW_ALL'
+            }
+            try:
+                results.append(
+                    client.execute_query(
+                        'get', verb_arguments=ref))
+            except HttpError:
+                results.append(r)
+        return results


Test patch:
diff --git a/tools/c7n_gcp/tests/data/flights/dataflow-job/get-v1b3-projects-cloud-custodian-jobs-2021-04-27_08_34_20-8562643317148312878_1.json b/tools/c7n_gcp/tests/data/flights/dataflow-job/get-v1b3-projects-cloud-custodian-jobs-2021-04-27_08_34_20-8562643317148312878_1.json
new file mode 100644
--- /dev/null
+++ b/tools/c7n_gcp/tests/data/flights/dataflow-job/get-v1b3-projects-cloud-custodian-jobs-2021-04-27_08_34_20-8562643317148312878_1.json
@@ -0,0 +1,1066 @@
+{
+  "headers": {
+    "status": "200", 
+    "content-length": "1193", 
+    "x-xss-protection": "0", 
+    "content-location": "https://dataflow.googleapis.com/v1b3/projects/cloud-custodian/jobs/2019-05-16_04_24_18-6110555549864901093?alt=json", 
+    "x-content-type-options": "nosniff", 
+    "transfer-encoding": "chunked", 
+    "vary": "Origin, X-Origin, Referer", 
+    "server": "ESF", 
+    "-content-encoding": "gzip", 
+    "cache-control": "private", 
+    "date": "Thu, 16 May 2019 12:08:45 GMT", 
+    "x-frame-options": "SAMEORIGIN", 
+    "alt-svc": "quic=\":443\"; ma=2592000; v=\"46,44,43,39\"", 
+    "content-type": "application/json; charset=UTF-8"
+  },
+  "body": {
+    "id": "2021-04-27_08_34_20-8562643317148312878",
+    "projectId": "cloud-custodian",
+    "name": "test",
+    "type": "JOB_TYPE_STREAMING",
+    "environment": {
+      "tempStoragePrefix": "storage.googleapis.com/tmp/tmp",
+      "clusterManagerApiService": "instancegroup.googleapis.com",
+      "experiments": [
+        "enable_streaming_pubsub_io_stackdriver_metrics",
+        "enable_throttled_based_rescaling",
+        "enable_streaming_service_billing",
+        "use_multi_hop_delegation",
+        "enable_billing_v_1_5",
+        "increase_igm_polling_interval",
+        "override_controller_service_account",
+        "merge_default_environment_in_per_pipeline_component",
+        "use_replica_pools",
+        "emit_autoscaling_rationales",
+        "use_host_networking",
+        "use_compute_esf_endpoint",
+        "shuffle_service_num_reserved_repartitioner_sources=200000",
+        "allow_portable_job_submission",
+        "emit_autoscaling_monitoring_events",
+        "use_workflow_job_manager_components_for_streaming",
+        "use_timer_backlog_in_streaming_autoscaling",
+        "shuffle_service_max_shuffle_log_stripes=15000",
+        "use_dataflow_service_account_in_igm",
+        "igm_stable_polling_delay_secs=120",
+        "use_worker_zone_chooser_by_default",
+        "enable_compute_default_service_account_org_policy",
+        "shuffle_service_use_partitioning_log_v3",
+        "autoscale_windmill_service_by_default",
+        "enable_streaming_scaling",
+        "use_ppm_for_windmill"
+      ],
+      "workerPools": [
+        {
+          "kind": "harness",
+          "numWorkers": 2,
+          "packages": [
+            {
+              "name": "teleport-all-bundled-_uGQoGj2e2zz5Uspavdbswu75ubbmt8_U5Z1LHjdAZo.jar",
+              "location": "storage.googleapis.com/dataflow-templates-libraries/2021-04-19-00_RC00/teleport-all-bundled-_uGQoGj2e2zz5Uspavdbswu75ubbmt8_U5Z1LHjdAZo.jar"
+            }
+          ],
+          "machineType": "n1-standard-1",
+          "teardownPolicy": "TEARDOWN_ALWAYS",
+          "diskSizeGb": 30,
+          "zone": "us-central1-f",
+          "onHostMaintenance": "MIGRATE",
+          "dataDisks": [
+            {
+              "sizeGb": 10
+            }
+          ],
+          "metadata": {
+            "job_name": "test",
+            "sdk_pipeline_options": "{\"display_data\":[{\"key\":\"jobName\",\"namespace\":\"org.apache.beam.sdk.options.PipelineOptions\",\"type\":\"STRING\",\"value\":\"pubsubtopubsub-dataflow0releaser-0419082627-8130e736\"},{\"key\":\"streaming\",\"namespace\":\"org.apache.beam.sdk.options.StreamingOptions\",\"type\":\"BOOLEAN\",\"value\":true},{\"key\":\"labels\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineOptions\",\"type\":\"STRING\",\"value\":\"{goog-dataflow-provided-template-name=cloud_pubsub_to_cloud_pubsub, goog-dataflow-provided-template-version=2021-04-19-00_rc00, goog-dataflow-provided-template-type=legacy}\"},{\"key\":\"autoscalingAlgorithm\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions\",\"type\":\"STRING\",\"value\":\"THROUGHPUT_BASED\"},{\"key\":\"stagingLocation\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineOptions\",\"type\":\"STRING\",\"value\":\"gs://dataflow-templates-libraries/2021-04-19-00_RC00\"},{\"key\":\"pipelineUrl\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineOptions\",\"type\":\"STRING\",\"value\":\"gs://dataflow-templates-libraries/2021-04-19-00_RC00/pipeline-T2clImR1XdMoJ4t-nPZUDhQdcflNPD6tuHFqqXCLTSA.pb\"},{\"key\":\"maxNumWorkers\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions\",\"type\":\"INTEGER\",\"value\":3},{\"key\":\"project\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineOptions\",\"type\":\"STRING\",\"value\":\"dataflow-templates\"},{\"key\":\"gcsUploadBufferSizeBytes\",\"namespace\":\"org.apache.beam.sdk.extensions.gcp.options.GcsOptions\",\"type\":\"INTEGER\",\"value\":1048576},{\"key\":\"userAgent\",\"namespace\":\"org.apache.beam.sdk.options.PipelineOptions\",\"type\":\"STRING\",\"value\":\"Apache_Beam_SDK_for_Java/2.27.0(JDK_11_environment)\"},{\"key\":\"tempLocation\",\"namespace\":\"org.apache.beam.sdk.options.PipelineOptions\",\"type\":\"STRING\",\"value\":\"gs://tf-state-service-accounts/tmp\"},{\"key\":\"appName\",\"namespace\":\"org.apache.beam.sdk.options.ApplicationNameOptions\",\"type\":\"STRING\",\"value\":\"PubsubToPubsub\"},{\"key\":\"region\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineOptions\",\"type\":\"STRING\",\"value\":\"us-central1\"},{\"key\":\"filesToStage\",\"namespace\":\"org.apache.beam.sdk.options.PortablePipelineOptions\",\"type\":\"STRING\",\"value\":\"[/export/hda3/borglet/remote_hdd_fs_dirs/0.rapid.runner-t2zfzvyb-xzn6-rgig-pryn-ouxnqent52mf.dataflow-releaser.798918173332.14b334fb3717c109/mount/rapid/workflows/e09982ac-8a02-4fd8-8bfc-39f83a073a9b/tmp/tmpxfq9vedq/teleport-all-bundled.jar]\"},{\"key\":\"templateLocation\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineOptions\",\"type\":\"STRING\",\"value\":\"gs://dataflow-templates-us-central1/latest/Cloud_PubSub_to_Cloud_PubSub\"},{\"key\":\"filesToStage\",\"namespace\":\"org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions\",\"type\":\"STRING\",\"value\":\"[/export/hda3/borglet/remote_hdd_fs_dirs/0.rapid.runner-t2zfzvyb-xzn6-rgig-pryn-ouxnqent52mf.dataflow-releaser.798918173332.14b334fb3717c109/mount/rapid/workflows/e09982ac-8a02-4fd8-8bfc-39f83a073a9b/tmp/tmpxfq9vedq/teleport-all-bundled.jar]\"},{\"key\":\"runner\",\"namespace\":\"org.apache.beam.sdk.options.PipelineOptions\",\"shortValue\":\"DataflowRunner\",\"type\":\"JAVA_CLASS\",\"value\":\"org.apache.beam.runners.dataflow.DataflowRunner\"},{\"key\":\"gcpTempLocation\",\"namespace\":\"google.dataflow.v1beta3.TemplatesService\",\"type\":\"STRING\",\"value\":\"gs://tf-state-service-accounts/tmp\"}],\"options\":{\"apiRootUrl\":\"https://dataflow.googleapis.com/\",\"appName\":\"PubsubToPubsub\",\"autoscalingAlgorithm\":\"BASIC\",\"credentialFactoryClass\":\"org.apache.beam.sdk.extensions.gcp.auth.GcpCredentialFactory\",\"dataflowEndpoint\":\"\",\"dataflowJobId\":\"2021-04-27_08_34_20-8562643317148312878\",\"dataflowKmsKey\":null,\"dataflowWorkerJar\":null,\"defaultEnvironmentConfig\":null,\"defaultEnvironmentType\":null,\"diskSizeGb\":0,\"enableCloudDebugger\":false,\"enableStreamingEngine\":false,\"experiments\":null,\"filesToStage\":[\"/export/hda3/borglet/remote_hdd_fs_dirs/0.rapid.runner-t2zfzvyb-xzn6-rgig-pryn-ouxnqent52mf.dataflow-releaser.798918173332.14b334fb3717c109/mount/rapid/workflows/e09982ac-8a02-4fd8-8bfc-39f83a073a9b/tmp/tmpxfq9vedq/teleport-all-bundled.jar\"],\"filterKey\":null,\"filterValue\":null,\"gcpTempLocation\":\"gs://tf-state-service-accounts/tmp\",\"gcsPerformanceMetrics\":false,\"gcsUploadBufferSizeBytes\":1048576,\"googleApiTrace\":null,\"inputSubscription\":\"projects/cloud-custodian/subscriptions/test\",\"jobName\":\"pubsubtopubsub-dataflow0releaser-0419082627-8130e736\",\"labels\":{\"goog-dataflow-provided-template-name\":\"cloud_pubsub_to_cloud_pubsub\",\"goog-dataflow-provided-template-type\":\"legacy\",\"goog-dataflow-provided-template-version\":\"2021-04-19-00_rc00\"},\"maxNumWorkers\":3,\"network\":null,\"numWorkers\":2,\"numberOfWorkerHarnessThreads\":0,\"optionsId\":0,\"outputTopic\":\"projects/cloud-custodian/topics/topic\",\"overrideWindmillBinary\":null,\"pathValidatorClass\":\"org.apache.beam.sdk.extensions.gcp.storage.GcsPathValidator\",\"pipelineUrl\":\"gs://dataflow-templates-libraries/2021-04-19-00_RC00/pipeline-T2clImR1XdMoJ4t-nPZUDhQdcflNPD6tuHFqqXCLTSA.pb\",\"project\":\"cloud-custodian\",\"region\":\"us-central1\",\"runner\":\"org.apache.beam.runners.dataflow.DataflowRunner\",\"saveProfilesToGcs\":null,\"serviceAccount\":null,\"stableUniqueNames\":\"WARNING\",\"stagerClass\":\"org.apache.beam.runners.dataflow.util.GcsStager\",\"stagingLocation\":\"gs://dataflow-templates-libraries/2021-04-19-00_RC00\",\"streaming\":true,\"subnetwork\":null,\"tempLocation\":\"gs://tf-state-service-accounts/tmp\",\"templateLocation\":\"gs://dataflow-templates-us-central1/latest/Cloud_PubSub_to_Cloud_PubSub\",\"userAgent\":\"Apache_Beam_SDK_for_Java/2.27.0(JDK_11_environment)\",\"workerDiskType\":null,\"workerHarnessContainerImage\":\"gcr.io/cloud-dataflow/v1beta3/IMAGE:beam-2.27.0\",\"workerMachineType\":null,\"workerRegion\":null,\"zone\":null}}",
+            "packages": "gs://dataflow-templates-libraries/2021-04-19-00_RC00/teleport-all-bundled-_uGQoGj2e2zz5Uspavdbswu75ubbmt8_U5Z1LHjdAZo.jar|teleport-all-bundled-_uGQoGj2e2zz5Uspavdbswu75ubbmt8_U5Z1LHjdAZo.jar"
+          },
+          "autoscalingSettings": {
+            "algorithm": "AUTOSCALING_ALGORITHM_BASIC",
+            "maxNumWorkers": 3
+          },
+          "poolArgs": {},
+          "network": "default",
+          "workerHarnessContainerImage": "gcr.io/cloud-dataflow/v1beta3/beam-java11-streaming:beam-2.27.0"
+        }
+      ],
+      "userAgent": {
+        "name": "Apache Beam SDK for Java",
+        "legacy.environment.major.version": "8",
+        "version": "2.27.0",
+        "os.name": "Linux",
+        "container.version": "beam-2.27.0",
+        "fnapi.environment.major.version": "8",
+        "os.arch": "amd64",
+        "java.version": "11.0.10",
+        "java.vendor": "Google Inc.",
+        "os.version": "4.15.0-smp-911.32.0.0"
+      },
+      "version": {
+        "major": "8",
+        "job_type": "STREAMING"
+      },
+      "dataset": "bigquery.googleapis.com/cloud_dataflow",
+      "sdkPipelineOptions": {
+        "display_data": [
+          {
+            "value": "pubsubtopubsub-dataflow0releaser-0419082627-8130e736",
+            "type": "STRING",
+            "namespace": "org.apache.beam.sdk.options.PipelineOptions",
+            "key": "jobName"
+          },
+          {
+            "type": "BOOLEAN",
+            "value": true,
+            "namespace": "org.apache.beam.sdk.options.StreamingOptions",
+            "key": "streaming"
+          },
+          {
+            "type": "STRING",
+            "key": "labels",
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+            "value": "{goog-dataflow-provided-template-name=cloud_pubsub_to_cloud_pubsub, goog-dataflow-provided-template-version=2021-04-19-00_rc00, goog-dataflow-provided-template-type=legacy}"
+          },
+          {
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions",
+            "key": "autoscalingAlgorithm",
+            "value": "THROUGHPUT_BASED",
+            "type": "STRING"
+          },
+          {
+            "value": "gs://dataflow-templates-libraries/2021-04-19-00_RC00",
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+            "key": "stagingLocation",
+            "type": "STRING"
+          },
+          {
+            "key": "pipelineUrl",
+            "type": "STRING",
+            "value": "gs://dataflow-templates-libraries/2021-04-19-00_RC00/pipeline-T2clImR1XdMoJ4t-nPZUDhQdcflNPD6tuHFqqXCLTSA.pb",
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions"
+          },
+          {
+            "key": "maxNumWorkers",
+            "value": 3,
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions",
+            "type": "INTEGER"
+          },
+          {
+            "key": "project",
+            "type": "STRING",
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+            "value": "dataflow-templates"
+          },
+          {
+            "key": "gcsUploadBufferSizeBytes",
+            "namespace": "org.apache.beam.sdk.extensions.gcp.options.GcsOptions",
+            "value": 1048576,
+            "type": "INTEGER"
+          },
+          {
+            "type": "STRING",
+            "key": "userAgent",
+            "namespace": "org.apache.beam.sdk.options.PipelineOptions",
+            "value": "Apache_Beam_SDK_for_Java/2.27.0(JDK_11_environment)"
+          },
+          {
+            "value": "gs://tf-state-service-accounts/tmp",
+            "key": "tempLocation",
+            "type": "STRING",
+            "namespace": "org.apache.beam.sdk.options.PipelineOptions"
+          },
+          {
+            "type": "STRING",
+            "key": "appName",
+            "namespace": "org.apache.beam.sdk.options.ApplicationNameOptions",
+            "value": "PubsubToPubsub"
+          },
+          {
+            "value": "us-central1",
+            "key": "region",
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+            "type": "STRING"
+          },
+          {
+            "value": "[/export/hda3/borglet/remote_hdd_fs_dirs/0.rapid.runner-t2zfzvyb-xzn6-rgig-pryn-ouxnqent52mf.dataflow-releaser.798918173332.14b334fb3717c109/mount/rapid/workflows/e09982ac-8a02-4fd8-8bfc-39f83a073a9b/tmp/tmpxfq9vedq/teleport-all-bundled.jar]",
+            "namespace": "org.apache.beam.sdk.options.PortablePipelineOptions",
+            "type": "STRING",
+            "key": "filesToStage"
+          },
+          {
+            "type": "STRING",
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+            "value": "gs://dataflow-templates-us-central1/latest/Cloud_PubSub_to_Cloud_PubSub",
+            "key": "templateLocation"
+          },
+          {
+            "type": "STRING",
+            "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions",
+            "value": "[/export/hda3/borglet/remote_hdd_fs_dirs/0.rapid.runner-t2zfzvyb-xzn6-rgig-pryn-ouxnqent52mf.dataflow-releaser.798918173332.14b334fb3717c109/mount/rapid/workflows/e09982ac-8a02-4fd8-8bfc-39f83a073a9b/tmp/tmpxfq9vedq/teleport-all-bundled.jar]",
+            "key": "filesToStage"
+          },
+          {
+            "key": "runner",
+            "namespace": "org.apache.beam.sdk.options.PipelineOptions",
+            "shortValue": "DataflowRunner",
+            "type": "JAVA_CLASS",
+            "value": "org.apache.beam.runners.dataflow.DataflowRunner"
+          },
+          {
+            "type": "STRING",
+            "namespace": "google.dataflow.v1beta3.TemplatesService",
+            "value": "gs://tf-state-service-accounts/tmp",
+            "key": "gcpTempLocation"
+          }
+        ],
+        "options": {
+          "filterKey": null,
+          "credentialFactoryClass": "org.apache.beam.sdk.extensions.gcp.auth.GcpCredentialFactory",
+          "filterValue": null,
+          "numWorkers": 0,
+          "labels": {
+            "goog-dataflow-provided-template-name": "cloud_pubsub_to_cloud_pubsub",
+            "goog-dataflow-provided-template-version": "2021-04-19-00_rc00",
+            "goog-dataflow-provided-template-type": "legacy"
+          },
+          "enableStreamingEngine": false,
+          "autoscalingAlgorithm": "THROUGHPUT_BASED",
+          "overrideWindmillBinary": null,
+          "saveProfilesToGcs": null,
+          "pathValidatorClass": "org.apache.beam.sdk.extensions.gcp.storage.GcsPathValidator",
+          "dataflowEndpoint": "",
+          "googleApiTrace": null,
+          "stagingLocation": "gs://dataflow-templates-libraries/2021-04-19-00_RC00",
+          "workerHarnessContainerImage": "gcr.io/cloud-dataflow/v1beta3/IMAGE:beam-2.27.0",
+          "filesToStage": [
+            "/export/hda3/borglet/remote_hdd_fs_dirs/0.rapid.runner-t2zfzvyb-xzn6-rgig-pryn-ouxnqent52mf.dataflow-releaser.798918173332.14b334fb3717c109/mount/rapid/workflows/e09982ac-8a02-4fd8-8bfc-39f83a073a9b/tmp/tmpxfq9vedq/teleport-all-bundled.jar"
+          ],
+          "runner": "org.apache.beam.runners.dataflow.DataflowRunner",
+          "region": "us-central1",
+          "project": "dataflow-templates",
+          "optionsId": 0,
+          "stagerClass": "org.apache.beam.runners.dataflow.util.GcsStager",
+          "dataflowKmsKey": null,
+          "diskSizeGb": 0,
+          "tempLocation": "gs://tf-state-service-accounts/tmp",
+          "appName": "PubsubToPubsub",
+          "workerDiskType": null,
+          "workerMachineType": null,
+          "enableCloudDebugger": false,
+          "gcpTempLocation": "gs://tf-state-service-accounts/tmp",
+          "zone": null,
+          "dataflowWorkerJar": null,
+          "gcsUploadBufferSizeBytes": 1048576,
+          "defaultEnvironmentType": null,
+          "subnetwork": null,
+          "workerRegion": null,
+          "outputTopic": null,
+          "serviceAccount": null,
+          "inputSubscription": null,
+          "numberOfWorkerHarnessThreads": 0,
+          "gcsPerformanceMetrics": false,
+          "maxNumWorkers": 3,
+          "userAgent": "Apache_Beam_SDK_for_Java/2.27.0(JDK_11_environment)",
+          "streaming": true,
+          "templateLocation": "gs://dataflow-templates-us-central1/latest/Cloud_PubSub_to_Cloud_PubSub",
+          "apiRootUrl": "https://dataflow.googleapis.com/",
+          "pipelineUrl": "gs://dataflow-templates-libraries/2021-04-19-00_RC00/pipeline-T2clImR1XdMoJ4t-nPZUDhQdcflNPD6tuHFqqXCLTSA.pb",
+          "network": null,
+          "jobName": "pubsubtopubsub-dataflow0releaser-0419082627-8130e736",
+          "defaultEnvironmentConfig": null,
+          "experiments": null,
+          "stableUniqueNames": "WARNING"
+        }
+      }
+    },
+    "steps": [
+      {
+        "kind": "ParallelRead",
+        "name": "s1",
+        "properties": {
+          "pubsub_serialized_attributes_fn": {
+            "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%00%e7%c8%02%c8%ac%ed%00%05sr%00Aorg.apache.beam.runners.dataflow.DataflowRu%01%19%80$IdentityMessageFn%df+D%103N%8c%f1%02%00%00xr%00->P%00%d0sdk.transforms.SimpleFunction%e0XCU8%12%02%13%02%00%01L%00%02fnt%005Lorg/%09%91%00/%01%91%10/sdk/%19A(/Serializab%19G%10;xr%000zy%00%10Infer.5%00%1cBw%f7%f9]+V%c8%19|%040Lz|%00LProcessFunction;xppp",
+            "@type": "http://schema.org/Text"
+          },
+          "format": {
+            "@type": "http://schema.org/Text",
+            "value": "pubsub"
+          },
+          "pubsub_subscription_runtime_override": {
+            "value": "inputSubscription",
+            "@type": "http://schema.org/Text"
+          },
+          "user_name": {
+            "value": "Read PubSub Events/PubsubUnboundedSource",
+            "@type": "http://schema.org/Text"
+          },
+          "output_info": [
+            {
+              "output_name": {
+                "value": "output",
+                "@type": "http://schema.org/Text"
+              },
+              "user_name": {
+                "@type": "http://schema.org/Text",
+                "value": "Read PubSub Events/PubsubUnboundedSource.out0"
+              },
+              "encoding": {
+                "@type": "kind:windowed_value",
+                "is_wrapper": {
+                  "value": true,
+                  "@type": "http://schema.org/Boolean"
+                },
+                "component_encodings": [
+                  {
+                    "serialized_coder": {
+                      "@type": "http://schema.org/Text",
+                      "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%00%92%bb%01%f0X%ac%ed%00%05sr%00Borg.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder%de%88%17?%e0%93%d6U%02%00%00xr%00&NQ%004coders.CustomC%01%0d j%b0%08%9d%0b;%1d%0b%02%055%00 n5%00%01/0C%dd%d5%89%ae%bc~%f8%02%00%00xp"
+                    },
+                    "type": {
+                      "@type": "http://schema.org/Text",
+                      "value": "org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder"
+                    },
+                    "@type": "org.apache.beam.sdk.coders.CustomCoder"
+                  },
+                  {
+                    "@type": "kind:global_window"
+                  }
+                ]
+              }
+            }
+          ]
+        }
+      },
+      {
+        "kind": "ParallelDo",
+        "name": "s2",
+        "properties": {
+          "display_data": [
+            {
+              "shortValue": {
+                "value": "PubsubIO$Read$$Lambda$133/0x0000000800d78440",
+                "@type": "http://schema.org/Text"
+              },
+              "namespace": {
+                "@type": "http://schema.org/Text",
+                "value": "org.apache.beam.sdk.transforms.MapElements"
+              },
+              "value": {
+                "@type": "http://schema.org/Text",
+                "value": "org.apache.beam.sdk.io.gcp.pubsub.PubsubIO$Read$$Lambda$133/0x0000000800d78440"
+              },
+              "key": {
+                "@type": "http://schema.org/Text",
+                "value": "class"
+              },
+              "type": {
+                "value": "JAVA_CLASS",
+                "@type": "http://schema.org/Text"
+              }
+            },
+            {
+              "namespace": {
+                "@type": "http://schema.org/Text",
+                "value": "org.apache.beam.sdk.transforms.ParDo$SingleOutput"
+              },
+              "type": {
+                "@type": "http://schema.org/Text",
+                "value": "JAVA_CLASS"
+              },
+              "label": {
+                "value": "Transform Function",
+                "@type": "http://schema.org/Text"
+              },
+              "value": {
+                "value": "org.apache.beam.sdk.transforms.MapElements$1",
+                "@type": "http://schema.org/Text"
+              },
+              "key": {
+                "@type": "http://schema.org/Text",
+                "value": "fn"
+              },
+              "shortValue": {
+                "@type": "http://schema.org/Text",
+                "value": ""
+              }
+            }
+          ],
+          "serialized_fn": {
+            "@type": "http://schema.org/Text",
+            "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%09G%ba%25%f0C%ac%ed%00%05sr%00!org.apache.beam.sdk.util.DoFnInfoU%02t%9cZ%d7]%d3%02%00%08L%00%04doFnt%00%25Lorg/a%057%00/%017</sdk/transforms/%01=%0c;L%00%15%01/%14Schema%01K(rmationt%006L%8a@%00B8%00@;L%00%0ainputCodert%00%22RF%00%00c%01%1d%04s/%05$%012$mainOutput^%b8%00Lvalues/TupleTag;L%00%0co%051%05D$st%00%0fLjava/!%170/Map;L%00%10sideI%01%8c8Mappingq%00~%00%05L%00%0e%15%18%1cViewst%00%14%09;%90lang/Iterable;L%00%11windowingStrategyt%00.R%cf%00%0d%9d%00W>0%00%18;xpsr%00,N%bf%019%88%90.MapElements$1%0e%05I%1e%f4%84%8d%b0%02%00%01L%00%06this$0t%00,Rw%00%19D%01%f2%11D%10;xr%00#zs%00!%fb(%acD#%06%ce%93%bd%9d%02%00%00%05%a7%00*z4%00%1d%a7%1c%d3`I%e6%a9%91h%c1%05%a5%14%02fnt%00+~%a1%00$Contextful%01%a0%00)zl%00%04PTQ%a7 ou%b8%af%8aC%ce%8c%03%01%a6%08xsr%82;%00%19i%5c%e6%22j%5c%cf%d4%d3%fd%02%00%02L%00%07closuret%00%12=%ee%14ObjectAM%14requir)%86%08t%00-~%cf%00%00R%1d/)%e6%00!A%7f%00.AD%f0F.invoke.SerializedLambdaoa%d0%94,)6%85%02%00%0aI%00%0eimplMethodKind[%00%0ccapturedArgst%00%13[N%a7%00%00%0e%09%25(ingClasst%00%11%1d&%05%13%1c;L%00%18funca%c4(alInterface%05%1c6%fc%00%04St%01H%0c;L%00%1dJ0%00%00M%05%9a%0cNamea5%0c%18L%00%22b%25%00%1cSignatur%11*%10%09impl%05p%0d;%1d%eb%1dQ%00%13%01'%09g>B%000%16instantiated%09#%08Typ%09`%1cxp%00%00%00%06ur1!)r-%c8%1c%90%ceX%9f%10s)lE%d3%0c%00%00%00%01%05%83%00%15%052%01%8d %1a%00%00%00%00vr%00/N%7f%028io.gcp.pubsub.P%05%078IO$Read%f6%f8%8c%f9@%db%90%97%01X%01H%0c%0ft%003z%1d%051%fd%81L%00F-%90,t%00%05applyt%00&(=%b8%0d%bd%00)F%13%00%00t%05%aa%00/%a9%fb%b9%c4%18io/gcp/%09%aa%00/2%aa%00%0ct%00%1clEmd$newBuilder$60328d98$1t%00d(R%db%02NS%00 Message;)%c22%00!$%04%25v%01%06%04%11t%85%ffB%c3%00%9d%bby%b1%08$Fn%014%0c!t%00%5cJ %01R%86%00^H%00%00$%0dVVV%01%81%7f%96%93%00%08t%00%14-P(fn$36334a93!H%00%8cVH%01%1d%90%18Process1%fd%00;F%df%01%00LzD%02%0d%c4%04fu!%1an%d2%00!6%10(sr%00+N%d8%02%19%93%00..%b6%04%1c>!%9aY%1dz0f%a5%c4%00%0a%f5%13%0cst%00%16%09%aa%e5J%10Colle%85M%89%e6%00%1f%85%e6%01%1b%00.%19%1bHs$EmptyListz%b8%17%b4<%a7%9e%dea*%c1%fc%00>z%92%00%10AutoV%e1%db%00_%c1%93BN%08%1c%e25%cb%d3%b7`%9a%ac%05%a5%04%11e%e9&%1cConverte%e1%f6%00%10%1d%ac%01%80%c1]%004zt%00Rj%00%1c%af%d5QB%d7}%01W%05%b9!P%100sr%00BNJ%00N(%04m+4WithAttributes%12%9a%08%1c%de%88%17?%e0%93%d6U%01X%08r%00&NQ%00%16%0d%09%18.Custom%055%1cj%b0%08%9d%0b;%1d%0b%095%00 n5%00%0eI%09%1cC%dd%d5%89%ae%bc~%f8%01/!uR%fe%07%16%a4%08%04.T%1aA%09D%b3%18yf[%c0z%b5%02%00%02Z%00%09gene%0e%e3%08%14dL%00%02id%01%ff%10%18xp%01tV%01%01%0dJ%00PY%01|.<init>:402#e1121b0805e8b107sr%00%11Y5%b8HashMap%05%07%da%c1%c3%16`%d1%03%00%02F%00%0aloadFactorI%00%09thresholdxp?@%a1%b0%1c%00%0cw%08%00%00%00%10%a1%d0%01%9a%00:%a5%d5%005%0e%0c%08%00%5c>%1a%01|vendor.guava.v26_0_jre.com.googl%05%0b%10mon.cI%d8%14.Immut%a1%a4%0cMap$%b1%b4%14edForm%05|,%00%00%00%02%00%02[%00%04key%05x%0c%16[%00%06)%01%01%8c%08%16xp%d9Y%19%0a%0csr%00j%ee%9b%00%05%9b%18Regular2%a2%00e+%00sb%a9%00$%01L%00%03mapt%00OB[%04)%1a%00/%25%1a%00/5%1a%10/com/)%1a%01%0b%0cmon/-%1a%00/.x%00%04;xe%13%00@^%cd%0a)%11%04.W>%00%0bH9%8a%9e%e7%08%0d%19%e0%02%00%0cZ%00%18allow%0e%0d%09TtenessSpecifiedZ%00%0dmode%1d%10D%1atimestampCombiner%1d%1d%14%10trigg%1d%13%08L%00%0f:[%00%08t%00%18%25%01%10joda/%01L%0c/Dur%12%d7%0c%0c;L%00%0f%0e%0a%0a4ingBehaviort%00A%05->f%07%b98%00/%22%09%0c%1a%e3%0b%0c$Clo.C%00<;L%00%0denvironmentIiB%08L%00%04%01%d8%08t%00?R`%00)5%0dR%00i&e%0c%1c$Accumul%05%b0%08Mod%0e%8b%0c%18%0eonTime%19%b3%00@RS%00n%b3%00%00O2B%00%0e%df%0c!%192e%01%08t%00<%a6W%00%01%902>%00%0c;L%00%07-%9b%08t%002%aaI%00)%cf%10;L%00%08wEA%0e%bd%0e%003%a6@%00Iv%00F%c1%a1a%af%0csr%00%16a|!%fc%00.%01%e3%00.1%fc%1c%00%00%02?zQ%ce%d6%a1'%08r%00%1f6%25%00 base.Base%1d.DY%19:%f4%8e%02%00%01J%00%07iMillis%12~%0a%05%01%0c~r%00?%01BI6%04.b%1e%97%0f9%83%00.%09%ce%0cing.%09%a3>6%02%05H$%00%00%00%12%00%00xr%00%0e%a5%25%12@%0c%08Enu%95~%01%1dXpt%00%11FIRE_IF_NON_EMPTYt%00%01%84%00=N%84%00IZ%0dvnZ%02%01~%01%01%01e%81%cf8St%00%16DISCARDING_%01t D_PANES~r%82%cb%07B%ef%006r%02%01h%01%01%1dl%00%0b%01a%14_ALWAY%01a%00:N%cc%00RP%01B|%02%01Y%01%01%1d]D%0dEND_OF_WINDOWsr%007%a2_%00%18DefaultM%99%1cj%136%d1%8a%16%8b%1fI7%000%a2F%00%0d?%1c%be,k%b7f%f5%9b%80%12%c3%08%0c%0bsub%0d%18%c5@%002%a9Y%0el%08%006%a2Y%00%14GlobalIT$s%96%16%b9%14%02%8a%af%0f%02EE%00;%a2E%00$NonMerging%09I$FnW%06%0bg%d3%ee%a8%ab%09J%001%a2J%00%11@0%c6%04%19y%ba%8a%96W%02%00%00xp"
+          },
+          "parallel_input": {
+            "step_name": "s1",
+            "output_name": "output",
+            "@type": "OutputReference"
+          },
+          "user_name": {
+            "@type": "http://schema.org/Text",
+            "value": "Read PubSub Events/MapElements/Map"
+          },
+          "user_fn": {
+            "value": "org.apache.beam.sdk.transforms.MapElements$1",
+            "@type": "http://schema.org/Text"
+          },
+          "output_info": [
+            {
+              "user_name": {
+                "value": "Read PubSub Events/MapElements/Map.out0",
+                "@type": "http://schema.org/Text"
+              },
+              "encoding": {
+                "component_encodings": [
+                  {
+                    "type": {
+                      "value": "org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder",
+                      "@type": "http://schema.org/Text"
+                    },
+                    "@type": "org.apache.beam.sdk.coders.CustomCoder",
+                    "serialized_coder": {
+                      "@type": "http://schema.org/Text",
+                      "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%00%92%bb%01%f0X%ac%ed%00%05sr%00Borg.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder%de%88%17?%e0%93%d6U%02%00%00xr%00&NQ%004coders.CustomC%01%0d j%b0%08%9d%0b;%1d%0b%02%055%00 n5%00%01/0C%dd%d5%89%ae%bc~%f8%02%00%00xp"
+                    }
+                  },
+                  {
+                    "@type": "kind:global_window"
+                  }
+                ],
+                "@type": "kind:windowed_value",
+                "is_wrapper": {
+                  "value": true,
+                  "@type": "http://schema.org/Boolean"
+                }
+              },
+              "output_name": {
+                "@type": "http://schema.org/Text",
+                "value": "org.apache.beam.sdk.values.PCollection.<init>:402#e1121b0805e8b107"
+              }
+            }
+          ]
+        }
+      },
+      {
+        "kind": "ParallelDo",
+        "name": "s3",
+        "properties": {
+          "output_info": [
+            {
+              "user_name": {
+                "value": "Filter Events If Enabled.out0",
+                "@type": "http://schema.org/Text"
+              },
+              "output_name": {
+                "@type": "http://schema.org/Text",
+                "value": "org.apache.beam.sdk.values.PCollection.<init>:402#f0f5b4f22d0cd0f1"
+              },
+              "encoding": {
+                "component_encodings": [
+                  {
+                    "@type": "org.apache.beam.sdk.coders.CustomCoder",
+                    "serialized_coder": {
+                      "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%00%92%bb%01%f0X%ac%ed%00%05sr%00Borg.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder%de%88%17?%e0%93%d6U%02%00%00xr%00&NQ%004coders.CustomC%01%0d j%b0%08%9d%0b;%1d%0b%02%055%00 n5%00%01/0C%dd%d5%89%ae%bc~%f8%02%00%00xp",
+                      "@type": "http://schema.org/Text"
+                    },
+                    "type": {
+                      "@type": "http://schema.org/Text",
+                      "value": "org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder"
+                    }
+                  },
+                  {
+                    "@type": "kind:global_window"
+                  }
+                ],
+                "@type": "kind:windowed_value",
+                "is_wrapper": {
+                  "value": true,
+                  "@type": "http://schema.org/Boolean"
+                }
+              }
+            }
+          ],
+          "user_fn": {
+            "@type": "http://schema.org/Text",
+            "value": "com.google.cloud.teleport.templates.AutoValue_PubsubToPubsub_ExtractAndFilterEventsFn"
+          },
+          "parallel_input": {
+            "step_name": "s2",
+            "output_name": "org.apache.beam.sdk.values.PCollection.<init>:402#e1121b0805e8b107",
+            "@type": "OutputReference"
+          },
+          "serialized_fn": {
+            "@type": "http://schema.org/Text",
+            "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%07%bc%e3%1c%f0C%ac%ed%00%05sr%00!org.apache.beam.sdk.util.DoFnInfoU%02t%9cZ%d7]%d3%02%00%08L%00%04doFnt%00%25Lorg/a%057%00/%017</sdk/transforms/%01=%0c;L%00%15%01/%14Schema%01K(rmationt%006L%8a@%00B8%00@;L%00%0ainputCodert%00%22RF%00%00c%01%1d%04s/%05$%012$mainOutput^%b8%00Lvalues/TupleTag;L%00%0co%051%05D$st%00%0fLjava/!%170/Map;L%00%10sideI%01%8c8Mappingq%00~%00%05L%00%0e%15%18%1cViewst%00%14%09;%90lang/Iterable;L%00%11windowingStrategyt%00.R%cf%00%0d%9d%00W>0%00%f0<;xpsr%00Ucom.google.cloud.teleport.templates.AutoValue_PubsubTo%09%08%a0_ExtractAndFilterEventsFn+S%af>%dd%a1 %15%02%00%02L%00%09fi%01%1c%14Keyt%00+R%a3%00%04op!%d6%04s/%05d%18Provide!{%00%0b%09:%05%17! %10%0axr%00K%8e%b1%00%09%9f%11%a7%00$^%a7%00<%e90%9f%09%aa%a5%ec%dd%02%00%04L%00%08do%09%c5%08t%00%13=t%14BooleaA;%04%0ei!%af%09$%05%cd%00%12%1d'%10Strin!%fa%00%15%1d&%05%b9%1cRegext%00%19%09-E%03%00r%01%13%18/Patter%01Z%18%11isNull%09[%055 q%00~%00%0cxr%00#N^%03y'ed0%acD#%06%ce%93%bd%9d%02%00%00xp%01%01%0csr%00>N8%00-g%00.%05b1g%1c$Runtime2%15%00 Y%e91%e7%d4:%88%8a%02A%93%1c%0cdefault%05%22%00t2%f6%00DObject;L%00%05klasst%00%11%09%e6E%ae%00C%01%13eA%1cethodNam%05%d7%0c%0dL%00%09%0d%8a%10Idt%00%10%1d2%04Lo%25C %0cproperty%153%18xppvr%00:%ca%fb%01%00OI[%00%00%19%01%1cxpt%00%0cget)n%18Keysr%00%0ea%9f%00.%01%b6%00.%05%840%8b%e4%90%cc%8f#%df%02%00%01J%00%05eA%0cxr%00%10%19%254Number%86%ac%95%1d%0b%94%e0%8b%25y%11a%00t]%f0%18sq%00~%00%11p%01%06%0c%17t%00%0e%15u%25J%01%16%04%1bt2%e4%02^%b9%01=%f1y%8b%a1%22B%e2%04p%e25%cb%d3%b7`%9a%ac%02%00%01L%00%11elementConverte%81%8a-oE%87 List;xr%004N-%02%1dtRj%00%1c%af%d5QB%d7}%01W%05%fd%0csr%00%1f%25%1e%01X%18.Collece%d3%14$Empty%01i%1cz%b8%17%b4<%a7%9e%de%110%00BNu%00 io.gcp.pu%81fm%c6PMessageWithAttributes%a5Y%1c%de%88%17?%e0%93%d6U%01S%08r%00&NQ%00%a9%cc%18.Custom%055%1cj%b0%08%9d%0b;%1d%0b%095%00 n5%00%c1%08%1cC%dd%d5%89%ae%bc~%f8%01/%01%e7R%91%03E%22%08s.T%cd%00D%b3%18yf[%c0z%b5%02%00%02Z%00%09gene%a1%a2%14dL%00%02id!%f6%10%0dxp%01tV%01%01%0dJ%00P9C|.<init>:402#f0f5b4f22d0cd0f1sr%00%119w%b8HashMap%05%07%da%c1%c3%16`%d1%03%00%02F%00%0aloadFactorI%00%09thresholdxp?@E%b1(%0cw%08%00%00%00%10%00%00%00%01%01%9a%00+%01%05%14)xsr%00%1e%19X%19%8c-%cf(MapY6%14%85Z%dc%e7%d01%17n/%00(Set%15%f5r%1d%b4%03%cb(%11/%00,Nw%01%0d%fcB%a9%06%b09%8a%9e%e7%08%0d%19%e0%02%00%0cZ%00%18allowedLatenessSpecifiedZ%00%0dmode%1d%10D%1atimestampCombiner%1d%1d%14%10trigg%1d%13%08L%00%0f:[%00%08t%00%18%c5%ae%10joda/%01L%0c/Dur%12%80%08T;L%00%0fclosingBehaviort%00A%05-%16%0a%09R%d3%08%f5%b2%ed%8c%04$C6C%00(;L%00%0denvirona%b6%00II,%08L%00%04%01%d8%08t%00?R`%00)5%0dR%00i&%0e%08%1c$Accumul%05%b0%08Mod%0e4%08%18%0eonTime%19%b3%00@RS%00y%faF%b3%00%00O2B%00%0e%88%08!%192e%01%08t%00<%a6W%00%01%902>%00%0c;L%00%07-%9b%08t%002%aaI%00)%cf%0c;L%00%08)h%10Fnt%003%a6@%00Iv%10Fn;xpa%19%0csr%00%16A%a4!%fc%00.%01%e3%00.1%fc%1c%00%00%02?zQ%ce%d6A%cb%08r%00%1f6%25%00 base.Base%1d.%10Y%19:%f4%8e%c5J%1c%07iMillis%09c%01%01%0c~r%00?%01BI6%04.b%1e@%0b9%83%00.%09%ce%0cing.%09%a3>6%02%01G%01%01%0c%12%00%00x2%c9%06%0cEnum%01%19%01%01%01%1dXpt%00%11FIRE_IF_NON_EMPTYt%00%01%84%00=N%84%00IZ%0dvnZ%02%01a%01%01%01e%81@8@t%00%16DISCARDING_%01t%1cD_PANES~Z%99%08n%ef%006r%02%01h%01%01%1dl%00%0b%01a%14_ALWAY%01a%00:N%cc%00Ra%00B|%02%01Y%01%01%1d]D%0dEND_OF_WINDOWsr%007%a2_%00%08Def%0e6%09M%99%1cj%136%d1%8a%16%8b%1fI7%000%a2F%00%0d?%1c%be,k%b7f%f5%9b%80%e5%d8%0c%0bsub%0d%18%12b%08%04!x%12c%08%10%25sr%006%a2Y%00%14GlobalIT$s%96%16%b9%14%02%8a%af%0f%02EE%00;%a2E%00$NonMerging%09I$FnW%06%0bg%d3%ee%a8%ab%09J%001%a2J%00%11@0%c6%04%19y%ba%8a%96W%02%00%00xp"
+          },
+          "user_name": {
+            "value": "Filter Events If Enabled",
+            "@type": "http://schema.org/Text"
+          },
+          "display_data": [
+            {
+              "shortValue": {
+                "value": "AutoValue_PubsubToPubsub_ExtractAndFilterEventsFn",
+                "@type": "http://schema.org/Text"
+              },
+              "label": {
+                "value": "Transform Function",
+                "@type": "http://schema.org/Text"
+              },
+              "key": {
+                "@type": "http://schema.org/Text",
+                "value": "fn"
+              },
+              "type": {
+                "@type": "http://schema.org/Text",
+                "value": "JAVA_CLASS"
+              },
+              "namespace": {
+                "@type": "http://schema.org/Text",
+                "value": "org.apache.beam.sdk.transforms.ParDo$SingleOutput"
+              },
+              "value": {
+                "@type": "http://schema.org/Text",
+                "value": "com.google.cloud.teleport.templates.AutoValue_PubsubToPubsub_ExtractAndFilterEventsFn"
+              }
+            }
+          ]
+        }
+      },
+      {
+        "kind": "ParallelDo",
+        "name": "s4",
+        "properties": {
+          "user_fn": {
+            "value": "org.apache.beam.sdk.transforms.MapElements$1",
+            "@type": "http://schema.org/Text"
+          },
+          "parallel_input": {
+            "step_name": "s3",
+            "output_name": "org.apache.beam.sdk.values.PCollection.<init>:402#f0f5b4f22d0cd0f1",
+            "@type": "OutputReference"
+          },
+          "serialized_fn": {
+            "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%09G%bc%25%f0C%ac%ed%00%05sr%00!org.apache.beam.sdk.util.DoFnInfoU%02t%9cZ%d7]%d3%02%00%08L%00%04doFnt%00%25Lorg/a%057%00/%017</sdk/transforms/%01=%0c;L%00%15%01/%14Schema%01K(rmationt%006L%8a@%00B8%00@;L%00%0ainputCodert%00%22RF%00%00c%01%1d%04s/%05$%012$mainOutput^%b8%00Lvalues/TupleTag;L%00%0co%051%05D$st%00%0fLjava/!%170/Map;L%00%10sideI%01%8c8Mappingq%00~%00%05L%00%0e%15%18%1cViewst%00%14%09;%90lang/Iterable;L%00%11windowingStrategyt%00.R%cf%00%0d%9d%00W>0%00%18;xpsr%00,N%bf%019%88%90.MapElements$1%0e%05I%1e%f4%84%8d%b0%02%00%01L%00%06this$0t%00,Rw%00%19D%01%f2%11D%10;xr%00#zs%00!%fb(%acD#%06%ce%93%bd%9d%02%00%00%05%a7%00*z4%00%1d%a7%1c%d3`I%e6%a9%91h%c1%05%a5%14%02fnt%00+~%a1%00$Contextful%01%a0%00)zl%00%04PTQ%a7 ou%b8%af%8aC%ce%8c%03%01%a6%08xsr%82;%00%19i%5c%e6%22j%5c%cf%d4%d3%fd%02%00%02L%00%07closuret%00%12=%ee%14ObjectAM%14requir)%86%08t%00-~%cf%00%00R%1d/)%e6%00!A%7f%00.AD%f0F.invoke.SerializedLambdaoa%d0%94,)6%85%02%00%0aI%00%0eimplMethodKind[%00%0ccapturedArgst%00%13[N%a7%00%00%0e%09%25(ingClasst%00%11%1d&%05%13%1c;L%00%18funca%c4(alInterface%05%1c6%fc%00%04St%01H%0c;L%00%1dJ0%00%00M%05%9a%0cNamea5%0c%18L%00%22b%25%00%1cSignatur%11*%10%09impl%05p%0d;%1d%eb%1dQ%00%13%01'%09g>B%000%16instantiated%09#%08Typ%09`%1cxp%00%00%00%06ur1!)r-%c8%1c%90%ceX%9f%10s)lE%d3%0c%00%00%00%01%05%83%00%15%052%01%8d %1a%00%00%00%00vr%000N%7f%028io.gcp.pubsub.P%05%07<IO$Write%06%86%d3%cb%8f%a3%ea%c1%01Y%01I%0c%0ft%003z%1e%051%fe%81M%00F-%91,t%00%05applyt%00&(=%b9%0d%be%00)F%13%00%00t%05%ab%00/%a9%fc%b9%c5%18io/gcp/%09%ab%00/6%ab%00%0ct%00%1clEod$newBuilder$cd095d61$1t%00d(R%dd%02NT%00 Message;)%c22%00!%25%04%25v%01%06%04%11t%a5%01B%c4%00%9d%bdy%b3%08$Fn%014%0c!t%00%5cJ!%01R%86%00^H%00%00$%0dVVW%01%81%81%96%93%00%08t%00%14-P(fn$36334a93!H%00%8cVH%01%1d%90%18Process1%fe%00;F%e0%01%00LzE%02%0d%c4%04fu!%1an%d2%00!6%10(sr%00+N%da%02%19%93%00..%b8%04%1c>!%9aY%1dz0f%a5%c6%00%0a%f5%15%0cst%00%16%09%aa%e5L%10Colle%85O%89%e8%00%1f%85%e8%01%1b%00.%19%1bHs$EmptyListz%b8%17%b4<%a7%9e%dea+%c1%fe%00>z%92%00%10AutoV%e1%dd%00_%c1%95BP%08%1c%e25%cb%d3%b7`%9a%ac%05%a5%04%11e%e9(%1cConverte%e1%f8%00%10%1d%ac%01%80%c1_%004zt%00Rj%00%1c%af%d5QB%d7}%01W%05%b9!P%100sr%00BNJ%00N*%04m+4WithAttributes%12%9c%08%1c%de%88%17?%e0%93%d6U%01X%08r%00&NQ%00%16%0f%09%18.Custom%055%1cj%b0%08%9d%0b;%1d%0b%095%00 n5%00%0eK%09%1cC%dd%d5%89%ae%bc~%f8%01/!uR%00%08%16%a6%08%04.T%1aC%09D%b3%18yf[%c0z%b5%02%00%02Z%00%09gene%0e%e5%08%14dL%00%02id%01%ff%10%18xp%01tV%01%01%0dJ%00PY%01h.<init>:402#4663620f501c927!G%00%11Y5%b8HashMap%05%07%da%c1%c3%16`%d1%03%00%02F%00%0aloadFactorI%00%09thresholdxp?@%a1%b2%1c%00%0cw%08%00%00%00%10%a1%d2%01%9a%00:%a5%d7%005%0e%0e%08%00%5c>%1a%01|vendor.guava.v26_0_jre.com.googl%05%0b%10mon.cI%d8%14.Immut%a1%a5%0cMap$%b1%b5%14edForm%05|,%00%00%00%02%00%02[%00%04key%05x%0c%16[%00%06)%01%01%8c%08%16xp%d9[%19%0a%0csr%00j%ee%9b%00%05%9b%18Regular2%a2%00e+%00sb%a9%00$%01L%00%03mapt%00OB[%04)%1a%00/%25%1a%00/5%1a%10/com/)%1a%01%0b%0cmon/-%1a%00/.x%00%04;xe%13%00@^%cf%0a)%11%04.W>%02%0bH9%8a%9e%e7%08%0d%19%e0%02%00%0cZ%00%18allow%0e%0f%09TtenessSpecifiedZ%00%0dmode%1d%10D%1atimestampCombiner%1d%1d%14%10trigg%1d%13%08L%00%0f:[%00%08t%00%18%25%01%10joda/%01L%0c/Dur%12%d9%0c%0c;L%00%0f%0e%0c%0a4ingBehaviort%00A%05->g%07%b98%00/%22%0b%0c%1a%e5%0b%0c$Clo.C%00<;L%00%0denvironmentIiB%08L%00%04%01%d8%08t%00?R`%00)5%0dR%00i&g%0c%1c$Accumul%05%b0%08Mod%0e%8d%0c%18%0eonTime%19%b3%00@RS%00n%b3%00%00O2B%00%0e%e1%0c!%192e%01%08t%00<%a6W%00%01%902>%00%0c;L%00%07-%9b%08t%002%aaI%00)%cf%10;L%00%08wEA%0e%bf%0e%003%a6@%00Iv%00F%c1%a1a%af%0csr%00%16a|!%fc%00.%01%e3%00.1%fc%1c%00%00%02?zQ%ce%d6%a1'%08r%00%1f6%25%00 base.Base%1d.DY%19:%f4%8e%02%00%01J%00%07iMillis%12%80%0a%05%01%0c~r%00?%01BI6%04.b%1e%99%0f9%83%00.%09%ce%0cing.%09%a3>6%02%05H$%00%00%00%12%00%00xr%00%0e%a5%25%12B%0c%08Enu%95~%01%1dXpt%00%11FIRE_IF_NON_EMPTYt%00%01%84%00=N%84%00IZ%0dvnZ%02%01~%01%01%01e%81%cf8St%00%16DISCARDING_%01t D_PANES~r%82%cb%07B%ef%006r%02%01h%01%01%1dl%00%0b%01a%14_ALWAY%01a%00:N%cc%00RP%01B|%02%01Y%01%01%1d]D%0dEND_OF_WINDOWsr%007%a2_%00%18DefaultM%99%1cj%136%d1%8a%16%8b%1fI7R=%0cV%a5%00I%d8%1c%be,k%b7f%f5%9b%80%12%c3%08%0c%0bsub%0d%18%c5@%002%a9Y%e1%25%006%a2%9f%00%14GlobalIT$s%96%16%b9%14%02%8a%af%0f%02EE%00;%a2E%00$NonMerging%09I$FnW%06%0bg%d3%ee%a8%ab%09J%001%a2J%00%11@0%c6%04%19y%ba%8a%96W%02%00%00xp",
+            "@type": "http://schema.org/Text"
+          },
+          "display_data": [
+            {
+              "shortValue": {
+                "value": "PubsubIO$Write$$Lambda$161/0x0000000800e1d040",
+                "@type": "http://schema.org/Text"
+              },
+              "namespace": {
+                "value": "org.apache.beam.sdk.transforms.MapElements",
+                "@type": "http://schema.org/Text"
+              },
+              "type": {
+                "value": "JAVA_CLASS",
+                "@type": "http://schema.org/Text"
+              },
+              "value": {
+                "value": "org.apache.beam.sdk.io.gcp.pubsub.PubsubIO$Write$$Lambda$161/0x0000000800e1d040",
+                "@type": "http://schema.org/Text"
+              },
+              "key": {
+                "value": "class",
+                "@type": "http://schema.org/Text"
+              }
+            },
+            {
+              "shortValue": {
+                "value": "",
+                "@type": "http://schema.org/Text"
+              },
+              "namespace": {
+                "@type": "http://schema.org/Text",
+                "value": "org.apache.beam.sdk.transforms.ParDo$SingleOutput"
+              },
+              "label": {
+                "@type": "http://schema.org/Text",
+                "value": "Transform Function"
+              },
+              "key": {
+                "value": "fn",
+                "@type": "http://schema.org/Text"
+              },
+              "type": {
+                "@type": "http://schema.org/Text",
+                "value": "JAVA_CLASS"
+              },
+              "value": {
+                "value": "org.apache.beam.sdk.transforms.MapElements$1",
+                "@type": "http://schema.org/Text"
+              }
+            }
+          ],
+          "output_info": [
+            {
+              "encoding": {
+                "@type": "kind:windowed_value",
+                "is_wrapper": {
+                  "value": true,
+                  "@type": "http://schema.org/Boolean"
+                },
+                "component_encodings": [
+                  {
+                    "@type": "org.apache.beam.sdk.coders.CustomCoder",
+                    "type": {
+                      "@type": "http://schema.org/Text",
+                      "value": "org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder"
+                    },
+                    "serialized_coder": {
+                      "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%00%92%bb%01%f0X%ac%ed%00%05sr%00Borg.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder%de%88%17?%e0%93%d6U%02%00%00xr%00&NQ%004coders.CustomC%01%0d j%b0%08%9d%0b;%1d%0b%02%055%00 n5%00%01/0C%dd%d5%89%ae%bc~%f8%02%00%00xp",
+                      "@type": "http://schema.org/Text"
+                    }
+                  },
+                  {
+                    "@type": "kind:global_window"
+                  }
+                ]
+              },
+              "user_name": {
+                "@type": "http://schema.org/Text",
+                "value": "Write PubSub Events/MapElements/Map.out0"
+              },
+              "output_name": {
+                "@type": "http://schema.org/Text",
+                "value": "org.apache.beam.sdk.values.PCollection.<init>:402#4663620f501c9270"
+              }
+            }
+          ],
+          "user_name": {
+            "@type": "http://schema.org/Text",
+            "value": "Write PubSub Events/MapElements/Map"
+          }
+        }
+      },
+      {
+        "kind": "ParallelWrite",
+        "name": "s5",
+        "properties": {
+          "parallel_input": {
+            "output_name": "org.apache.beam.sdk.values.PCollection.<init>:402#4663620f501c9270",
+            "@type": "OutputReference",
+            "step_name": "s4"
+          },
+          "encoding": {
+            "is_wrapper": {
+              "@type": "http://schema.org/Boolean",
+              "value": true
+            },
+            "@type": "kind:windowed_value",
+            "component_encodings": [
+              {
+                "@type": "org.apache.beam.sdk.coders.VoidCoder"
+              },
+              {
+                "@type": "kind:global_window"
+              }
+            ]
+          },
+          "pubsub_serialized_attributes_fn": {
+            "value": "%82SNAPPY%00%00%00%00%01%00%00%00%01%00%00%00%e7%c8%02%c8%ac%ed%00%05sr%00Aorg.apache.beam.runners.dataflow.DataflowRu%01%19%80$IdentityMessageFn%df+D%103N%8c%f1%02%00%00xr%00->P%00%d0sdk.transforms.SimpleFunction%e0XCU8%12%02%13%02%00%01L%00%02fnt%005Lorg/%09%91%00/%01%91%10/sdk/%19A(/Serializab%19G%10;xr%000zy%00%10Infer.5%00%1cBw%f7%f9]+V%c8%19|%040Lz|%00LProcessFunction;xppp",
+            "@type": "http://schema.org/Text"
+          },
+          "user_name": {
+            "@type": "http://schema.org/Text",
+            "value": "Write PubSub Events/PubsubUnboundedSink"
+          },
+          "pubsub_topic_runtime_override": {
+            "value": "outputTopic",
+            "@type": "http://schema.org/Text"
+          },
+          "format": {
+            "@type": "http://schema.org/Text",
+            "value": "pubsub"
+          }
+        }
+      }
+    ],
+    "currentState": "JOB_STATE_RUNNING",
+    "currentStateTime": "2021-04-27T15:34:29.003405Z",
+    "createTime": "2021-04-27T15:34:21.567761Z",
+    "labels": {
+      "goog-dataflow-provided-template-name": "cloud_pubsub_to_cloud_pubsub",
+      "goog-dataflow-provided-template-type": "legacy",
+      "goog-dataflow-provided-template-version": "2021-04-19-00_rc00"
+    },
+    "location": "us-central1",
+    "pipelineDescription": {
+      "originalPipelineTransform": [
+        {
+          "id": "s5",
+          "name": "Write PubSub Events/PubsubUnboundedSink",
+          "inputCollectionName": [
+            "Write PubSub Events/MapElements/Map.out0"
+          ]
+        },
+        {
+          "kind": "READ_KIND",
+          "id": "s1",
+          "name": "Read PubSub Events/PubsubUnboundedSource",
+          "outputCollectionName": [
+            "Read PubSub Events/PubsubUnboundedSource.out0"
+          ]
+        },
+        {
+          "kind": "PAR_DO_KIND",
+          "id": "s4",
+          "name": "Write PubSub Events/MapElements/Map",
+          "displayData": [
+            {
+              "key": "class",
+              "namespace": "org.apache.beam.sdk.transforms.MapElements",
+              "strValue": "org.apache.beam.sdk.io.gcp.pubsub.PubsubIO$Write$$Lambda$161/0x0000000800e1d040",
+              "shortStrValue": "PubsubIO$Write$$Lambda$161/0x0000000800e1d040"
+            },
+            {
+              "key": "fn",
+              "namespace": "org.apache.beam.sdk.transforms.ParDo$SingleOutput",
+              "strValue": "org.apache.beam.sdk.transforms.MapElements$1",
+              "label": "Transform Function"
+            }
+          ],
+          "outputCollectionName": [
+            "Write PubSub Events/MapElements/Map.out0"
+          ],
+          "inputCollectionName": [
+            "Filter Events If Enabled.out0"
+          ]
+        },
+        {
+          "kind": "PAR_DO_KIND",
+          "id": "s2",
+          "name": "Read PubSub Events/MapElements/Map",
+          "displayData": [
+            {
+              "key": "class",
+              "namespace": "org.apache.beam.sdk.transforms.MapElements",
+              "strValue": "org.apache.beam.sdk.io.gcp.pubsub.PubsubIO$Read$$Lambda$133/0x0000000800d78440",
+              "shortStrValue": "PubsubIO$Read$$Lambda$133/0x0000000800d78440"
+            },
+            {
+              "key": "fn",
+              "namespace": "org.apache.beam.sdk.transforms.ParDo$SingleOutput",
+              "strValue": "org.apache.beam.sdk.transforms.MapElements$1",
+              "label": "Transform Function"
+            }
+          ],
+          "outputCollectionName": [
+            "Read PubSub Events/MapElements/Map.out0"
+          ],
+          "inputCollectionName": [
+            "Read PubSub Events/PubsubUnboundedSource.out0"
+          ]
+        },
+        {
+          "kind": "PAR_DO_KIND",
+          "id": "s3",
+          "name": "Filter Events If Enabled",
+          "displayData": [
+            {
+              "key": "fn",
+              "namespace": "org.apache.beam.sdk.transforms.ParDo$SingleOutput",
+              "strValue": "com.google.cloud.teleport.templates.AutoValue_PubsubToPubsub_ExtractAndFilterEventsFn",
+              "shortStrValue": "AutoValue_PubsubToPubsub_ExtractAndFilterEventsFn",
+              "label": "Transform Function"
+            }
+          ],
+          "outputCollectionName": [
+            "Filter Events If Enabled.out0"
+          ],
+          "inputCollectionName": [
+            "Read PubSub Events/MapElements/Map.out0"
+          ]
+        }
+      ],
+      "executionPipelineStage": [
+        {
+          "name": "F0",
+          "id": "S01",
+          "kind": "PAR_DO_KIND",
+          "componentTransform": [
+            {
+              "userName": "Read PubSub Events/PubsubUnboundedSource",
+              "name": "s1",
+              "originalTransform": "Read PubSub Events/PubsubUnboundedSource"
+            },
+            {
+              "userName": "Read PubSub Events/MapElements/Map",
+              "name": "s2",
+              "originalTransform": "Read PubSub Events/MapElements/Map"
+            },
+            {
+              "userName": "Filter Events If Enabled",
+              "name": "s3",
+              "originalTransform": "Filter Events If Enabled"
+            },
+            {
+              "userName": "Write PubSub Events/MapElements/Map",
+              "name": "s4",
+              "originalTransform": "Write PubSub Events/MapElements/Map"
+            },
+            {
+              "userName": "Write PubSub Events/PubsubUnboundedSink",
+              "name": "s5",
+              "originalTransform": "Write PubSub Events/PubsubUnboundedSink"
+            }
+          ],
+          "componentSource": [
+            {
+              "userName": "Read PubSub Events/PubsubUnboundedSource-out0",
+              "name": "s1.output",
+              "originalTransformOrCollection": "Read PubSub Events/PubsubUnboundedSource.out0"
+            },
+            {
+              "userName": "Read PubSub Events/MapElements/Map-out0",
+              "name": "s2.org.apache.beam.sdk.values.PCollection.<init>:402#e1121b0805e8b107",
+              "originalTransformOrCollection": "Read PubSub Events/MapElements/Map.out0"
+            },
+            {
+              "userName": "Filter Events If Enabled-out0",
+              "name": "s3.org.apache.beam.sdk.values.PCollection.<init>:402#f0f5b4f22d0cd0f1",
+              "originalTransformOrCollection": "Filter Events If Enabled.out0"
+            },
+            {
+              "userName": "Write PubSub Events/MapElements/Map-out0",
+              "name": "s4.org.apache.beam.sdk.values.PCollection.<init>:402#4663620f501c9270",
+              "originalTransformOrCollection": "Write PubSub Events/MapElements/Map.out0"
+            }
+          ]
+        }
+      ],
+      "displayData": [
+        {
+          "key": "jobName",
+          "namespace": "org.apache.beam.sdk.options.PipelineOptions",
+          "strValue": "test"
+        },
+        {
+          "key": "streaming",
+          "namespace": "org.apache.beam.sdk.options.StreamingOptions",
+          "boolValue": true
+        },
+        {
+          "key": "labels",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+          "strValue": "{goog-dataflow-provided-template-name=cloud_pubsub_to_cloud_pubsub, goog-dataflow-provided-template-version=2021-04-19-00_rc00, goog-dataflow-provided-template-type=legacy}"
+        },
+        {
+          "key": "autoscalingAlgorithm",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions",
+          "strValue": "THROUGHPUT_BASED"
+        },
+        {
+          "key": "stagingLocation",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+          "strValue": "gs://dataflow-templates-libraries/2021-04-19-00_RC00"
+        },
+        {
+          "key": "pipelineUrl",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+          "strValue": "gs://dataflow-templates-libraries/2021-04-19-00_RC00/pipeline-T2clImR1XdMoJ4t-nPZUDhQdcflNPD6tuHFqqXCLTSA.pb"
+        },
+        {
+          "key": "maxNumWorkers",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions",
+          "int64Value": "3"
+        },
+        {
+          "key": "project",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+          "strValue": "cloud-custodian"
+        },
+        {
+          "key": "gcsUploadBufferSizeBytes",
+          "namespace": "org.apache.beam.sdk.extensions.gcp.options.GcsOptions",
+          "int64Value": "1048576"
+        },
+        {
+          "key": "userAgent",
+          "namespace": "org.apache.beam.sdk.options.PipelineOptions",
+          "strValue": "Apache_Beam_SDK_for_Java/2.27.0(JDK_11_environment)"
+        },
+        {
+          "key": "tempLocation",
+          "namespace": "org.apache.beam.sdk.options.PipelineOptions",
+          "strValue": "gs://tf-state-service-accounts/tmp"
+        },
+        {
+          "key": "appName",
+          "namespace": "org.apache.beam.sdk.options.ApplicationNameOptions",
+          "strValue": "PubsubToPubsub"
+        },
+        {
+          "key": "region",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+          "strValue": "us-central1"
+        },
+        {
+          "key": "filesToStage",
+          "namespace": "org.apache.beam.sdk.options.PortablePipelineOptions",
+          "strValue": "[/export/hda3/borglet/remote_hdd_fs_dirs/0.rapid.runner-t2zfzvyb-xzn6-rgig-pryn-ouxnqent52mf.dataflow-releaser.798918173332.14b334fb3717c109/mount/rapid/workflows/e09982ac-8a02-4fd8-8bfc-39f83a073a9b/tmp/tmpxfq9vedq/teleport-all-bundled.jar]"
+        },
+        {
+          "key": "templateLocation",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineOptions",
+          "strValue": "gs://dataflow-templates-us-central1/latest/Cloud_PubSub_to_Cloud_PubSub"
+        },
+        {
+          "key": "filesToStage",
+          "namespace": "org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions",
+          "strValue": "[/export/hda3/borglet/remote_hdd_fs_dirs/0.rapid.runner-t2zfzvyb-xzn6-rgig-pryn-ouxnqent52mf.dataflow-releaser.798918173332.14b334fb3717c109/mount/rapid/workflows/e09982ac-8a02-4fd8-8bfc-39f83a073a9b/tmp/tmpxfq9vedq/teleport-all-bundled.jar]"
+        },
+        {
+          "key": "runner",
+          "namespace": "org.apache.beam.sdk.options.PipelineOptions",
+          "strValue": "org.apache.beam.runners.dataflow.DataflowRunner",
+          "shortStrValue": "DataflowRunner"
+        },
+        {
+          "key": "gcpTempLocation",
+          "namespace": "google.dataflow.v1beta3.TemplatesService",
+          "strValue": "gs://tf-state-service-accounts/tmp"
+        },
+        {
+          "key": "outputTopic",
+          "strValue": "projects/cloud-custodian/topics/topic"
+        },
+        {
+          "key": "inputSubscription",
+          "strValue": "projects/cloud-custodian/subscriptions/test"
+        }
+      ]
+    },
+    "stageStates": [
+      {
+        "executionStageName": "move_attach_disks_step7",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "pause_stop_computation_step9",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "init_attach_disk_step2",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "init_start_computation_step3",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "move_start_computation_step8",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "failure18",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "setup_windmill1",
+        "executionStageState": "JOB_STATE_DONE",
+        "currentStateTime": "2021-04-27T15:34:29.450Z"
+      },
+      {
+        "executionStageName": "setup_resource_disks_harness13",
+        "executionStageState": "JOB_STATE_DONE",
+        "currentStateTime": "2021-04-27T15:34:29.396Z"
+      },
+      {
+        "executionStageName": "topology_move_in_disk_input_step",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "start19",
+        "executionStageState": "JOB_STATE_DONE",
+        "currentStateTime": "2021-04-27T15:34:29.239Z"
+      },
+      {
+        "executionStageName": "move_stop_computation_step5",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "topology_init_attach_disk_input_step",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "success17",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "setup_resource_projects/cloud-custodian/subscriptions/test-filter15",
+        "executionStageState": "JOB_STATE_DONE",
+        "currentStateTime": "2021-04-27T15:34:29.365Z"
+      },
+      {
+        "executionStageName": "teardown_resource_disks_harness14",
+        "executionStageState": "JOB_STATE_DONE",
+        "currentStateTime": "2021-04-27T15:34:29.482Z"
+      },
+      {
+        "executionStageName": "move_wait_windmill_setup_step4",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "move_detach_disks_step6",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "teardown_resource_projects/cloud-custodian/subscriptions/test-filter16",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "setup_resource_global_gce_worker_pool11",
+        "executionStageState": "JOB_STATE_DONE",
+        "currentStateTime": "2021-04-27T15:34:29.300Z"
+      },
+      {
+        "executionStageName": "topology_pause_input_step",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "topology_move_out_disk_input_step",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      },
+      {
+        "executionStageName": "F0",
+        "executionStageState": "JOB_STATE_RUNNING",
+        "currentStateTime": "2021-04-27T15:34:29.394Z"
+      },
+      {
+        "executionStageName": "teardown_resource_global_gce_worker_pool12",
+        "executionStageState": "JOB_STATE_DONE",
+        "currentStateTime": "2021-04-27T15:34:29.362Z"
+      },
+      {
+        "executionStageName": "pause_detach_disk_step10",
+        "executionStageState": "JOB_STATE_PENDING",
+        "currentStateTime": "2021-04-27T15:34:28.992Z"
+      }
+    ],
+    "jobMetadata": {
+      "sdkVersion": {
+        "version": "2.27.0",
+        "versionDisplayName": "Apache Beam SDK for Java",
+        "sdkSupportStatus": "SUPPORTED"
+      }
+    },
+    "startTime": "2021-04-27T15:34:21.567761Z"
+  }
+}
\ No newline at end of file
diff --git a/tools/c7n_gcp/tests/data/flights/dataflow-job/get-v1b3-projects-cloud-custodian-jobs-aggregated_1.json b/tools/c7n_gcp/tests/data/flights/dataflow-job/get-v1b3-projects-cloud-custodian-jobs-aggregated_1.json
--- a/tools/c7n_gcp/tests/data/flights/dataflow-job/get-v1b3-projects-cloud-custodian-jobs-aggregated_1.json
+++ b/tools/c7n_gcp/tests/data/flights/dataflow-job/get-v1b3-projects-cloud-custodian-jobs-aggregated_1.json
@@ -1,40 +1,25 @@
 {
+  "headers":{},
   "body": {
     "jobs": [
       {
-        "name": "test", 
+        "id": "2021-04-27_08_34_20-8562643317148312878",
         "projectId": "cloud-custodian",
-        "createTime": "2019-03-06T15:01:50.876626Z", 
+        "name": "test",
+        "type": "JOB_TYPE_STREAMING",
+        "currentState": "JOB_STATE_RUNNING",
+        "currentStateTime": "2021-04-27T15:34:29.003405Z",
+        "createTime": "2021-04-27T15:34:21.567761Z",
+        "location": "us-central1",
         "jobMetadata": {
           "sdkVersion": {
-            "versionDisplayName": "Apache Beam SDK for Java", 
-            "version": "2.10.0", 
+            "version": "2.27.0",
+            "versionDisplayName": "Apache Beam SDK for Java",
             "sdkSupportStatus": "SUPPORTED"
           }
-        }, 
-        "currentStateTime": "2019-03-06T15:01:52.612535Z", 
-        "startTime": "2019-03-06T15:01:50.876626Z", 
-        "type": "JOB_TYPE_BATCH", 
-        "id": "2019-03-06_07_01_49-7812926814622315875", 
-        "currentState": "JOB_STATE_FAILED", 
-        "location": "us-central1"
+        },
+        "startTime": "2021-04-27T15:34:21.567761Z"
       }
     ]
-  }, 
-  "headers": {
-    "status": "200", 
-    "content-length": "614", 
-    "x-xss-protection": "1; mode=block", 
-    "content-location": "https://dataflow.googleapis.com/v1b3/projects/cloud-custodian/jobs?alt=json",
-    "x-content-type-options": "nosniff", 
-    "transfer-encoding": "chunked", 
-    "vary": "Origin, X-Origin, Referer", 
-    "server": "ESF", 
-    "-content-encoding": "gzip", 
-    "cache-control": "private", 
-    "date": "Wed, 06 Mar 2019 15:20:07 GMT", 
-    "x-frame-options": "SAMEORIGIN", 
-    "alt-svc": "quic=\":443\"; ma=2592000; v=\"44,43,39\"", 
-    "content-type": "application/json; charset=UTF-8"
   }
-}
+}
\ No newline at end of file
diff --git a/tools/c7n_gcp/tests/test_dataflow.py b/tools/c7n_gcp/tests/test_dataflow.py
--- a/tools/c7n_gcp/tests/test_dataflow.py
+++ b/tools/c7n_gcp/tests/test_dataflow.py
@@ -18,6 +18,7 @@ def test_query(self):
         self.assertEqual(resource[0]['name'], 'test')
         self.assertEqual(resource[0]['projectId'], project_id)
         self.assertEqual(resource[0]['location'], 'us-central1')
+        self.assertTrue(resource[0].get("environment"))
 
     def test_job_get(self):
         project_id = 'cloud-custodian'