# Buggy code
def _upload_dependencies_to_object_store(
    self, runtime_configuration, pipeline_name, operation
):
    operation_artifact_archive = self._get_dependency_archive_name(operation)
    cos_directory = pipeline_name
    # upload operation dependencies to object store
    try:
        t0 = time.time()
        dependency_archive_path = self._generate_dependency_archive(operation)
        self.log_pipeline_info(
            pipeline_name,
            f"generated dependency archive: {dependency_archive_path}",
            operation_name=operation.name,
            duration=(time.time() - t0),
        )

        cos_client = CosClient(config=runtime_configuration)

        t0 = time.time()
        cos_client.upload_file_to_dir(
            dir=cos_directory,
            file_name=operation_artifact_archive,
            file_path=dependency_archive_path,
        )
        self.log_pipeline_info(
            pipeline_name,
            f"uploaded dependency archive to: {cos_directory}/{operation_artifact_archive}",
            operation_name=operation.name,
            duration=(time.time() - t0),
        )

    except FileNotFoundError as ex:
        self.log.error(
            "Dependencies were not found building archive for operation: {}".format(
                operation.name
            ),
            exc_info=True,
        )
        raise FileNotFoundError(
            "Node '{}' referenced dependencies that were not found: {}".format(
                operation.name, ex
            )
        )

    except BaseException as ex:
        self.log.error(
            "Error uploading artifacts to object storage for operation: {}".format(
                operation.name
            ),
            exc_info=True,
        )
        raise ex from ex


# Traceback (urllib3.exceptions.NewConnectionError)
[I 2021-03-04 17:28:22.928 ServerApp] Creating pipeline definition as a .py file
[I 2021-03-04 17:28:22.949 ServerApp] airflow 'hello_world_apache_airflow-0304172822' - processing pipeline dependencies to: https://minio-service.kubernetes:9000 bucket: airflow-pipeline-artifacts folder: hello_world_apache_airflow-0304172822
[I 2021-03-04 17:28:22.949 ServerApp] airflow 'hello_world_apache_airflow-0304172822':'load_data' - processing operation dependencies for id: a1de79c3-2be4-4fc3-ab86-4959b5291252
[I 2021-03-04 17:28:22.951 ServerApp] airflow 'hello_world_apache_airflow-0304172822':'load_data' - generated dependency archive: /var/folders/h8/l3qh8v192rv_vvggc6wyphzc0000gn/T/elyra/load_data-a1de79c3-2be4-4fc3-ab86-4959b5291252.tar.gz (0.002 secs)
Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd51ee6b950>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /airflow-pipeline-artifacts?location=
Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd51eed9a10>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /airflow-pipeline-artifacts?location=
Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd51ee9e710>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /airflow-pipeline-artifacts?location=
Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd51ee9ef90>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /airflow-pipeline-artifacts?location=
Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd51ee99310>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /airflow-pipeline-artifacts?location=
[E 2021-03-04 17:28:28.964 ServerApp] Error uploading artifacts to object storage for operation: load_data
Traceback (most recent call last):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connection.py", line 170, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/util/connection.py", line 73, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/socket.py", line 752, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 8] nodename nor servname provided, or not known

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 706, in urlopen
chunked=chunked,
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 382, in _make_request
self._validate_conn(conn)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 1010, in _validate_conn
conn.connect()
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connection.py", line 353, in connect
conn = self._new_conn()
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connection.py", line 182, in _new_conn
self, "Failed to establish a new connection: %s" % e
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7fd51ee36210>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor.py", line 218, in _upload_dependencies_to_object_store
cos_client = CosClient(config=runtime_configuration)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/util/cos.py", line 41, in __init__
self.client = self.__initialize_object_store()
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/util/cos.py", line 53, in __initialize_object_store
if not self.client.bucket_exists(self.bucket):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 404, in bucket_exists
self._url_open('HEAD', bucket_name=bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2189, in _url_open
region = self._get_bucket_region(bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2067, in _get_bucket_region
region = self._get_bucket_location(bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2102, in _get_bucket_location
headers=headers)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/poolmanager.py", line 375, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
[Previous line repeated 2 more times]
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 756, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/util/retry.py", line 573, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='minio-service.kubernetes', port=9000): Max retries exceeded with url: /airflow-pipeline-artifacts?location= (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd51ee36210>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))
[E 2021-03-04 17:28:28.972 ServerApp] Uncaught exception POST /elyra/pipeline/export?1614907702924 (::1)
HTTPServerRequest(protocol='http', host='localhost:8888', method='POST', uri='/elyra/pipeline/export?1614907702924', version='HTTP/1.1', remote_ip='::1')
Traceback (most recent call last):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/tornado/web.py", line 1704, in _execute
result = await result
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/handlers.py", line 54, in post
pipeline_overwrite
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor.py", line 86, in export
None, processor.export, pipeline, pipeline_export_format, pipeline_export_path, overwrite)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor_airflow.py", line 123, in export
pipeline_name=pipeline_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor_airflow.py", line 256, in create_pipeline_file
notebook_ops = self._cc_pipeline(pipeline, pipeline_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor_airflow.py", line 219, in _cc_pipeline
operation)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor.py", line 238, in _upload_dependencies_to_object_store
raise ex from ex
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor.py", line 218, in _upload_dependencies_to_object_store
cos_client = CosClient(config=runtime_configuration)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/util/cos.py", line 41, in __init__
self.client = self.__initialize_object_store()
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/util/cos.py", line 53, in __initialize_object_store
if not self.client.bucket_exists(self.bucket):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 404, in bucket_exists
self._url_open('HEAD', bucket_name=bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2189, in _url_open
region = self._get_bucket_region(bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2067, in _get_bucket_region
region = self._get_bucket_location(bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2102, in _get_bucket_location
headers=headers)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/poolmanager.py", line 375, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
[Previous line repeated 2 more times]
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 756, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/util/retry.py", line 573, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='minio-service.kubernetes', port=9000): Max retries exceeded with url: /airflow-pipeline-artifacts?location= (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd51ee36210>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))
[E 2021-03-04 17:28:28.975 ServerApp] Uncaught exception in write_error
Traceback (most recent call last):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/tornado/web.py", line 1704, in _execute
result = await result
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/handlers.py", line 54, in post
pipeline_overwrite
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor.py", line 86, in export
None, processor.export, pipeline, pipeline_export_format, pipeline_export_path, overwrite)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor_airflow.py", line 123, in export
pipeline_name=pipeline_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor_airflow.py", line 256, in create_pipeline_file
notebook_ops = self._cc_pipeline(pipeline, pipeline_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor_airflow.py", line 219, in _cc_pipeline
operation)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor.py", line 238, in _upload_dependencies_to_object_store
raise ex from ex
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/pipeline/processor.py", line 218, in _upload_dependencies_to_object_store
cos_client = CosClient(config=runtime_configuration)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/util/cos.py", line 41, in __init__
self.client = self.__initialize_object_store()
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/util/cos.py", line 53, in __initialize_object_store
if not self.client.bucket_exists(self.bucket):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 404, in bucket_exists
self._url_open('HEAD', bucket_name=bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2189, in _url_open
region = self._get_bucket_region(bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2067, in _get_bucket_region
region = self._get_bucket_location(bucket_name)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/minio/api.py", line 2102, in _get_bucket_location
headers=headers)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/poolmanager.py", line 375, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 796, in urlopen
**response_kw
[Previous line repeated 2 more times]
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/connectionpool.py", line 756, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/urllib3/util/retry.py", line 573, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='minio-service.kubernetes', port=9000): Max retries exceeded with url: /airflow-pipeline-artifacts?location= (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd51ee36210>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/tornado/web.py", line 1217, in send_error
self.write_error(status_code, **kwargs)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/elyra/util/http.py", line 75, in write_error
self.set_status(status_code, reason=reply['reason'])
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/tornado/web.py", line 358, in set_status
self._reason = escape.native_str(reason)
File "/opt/anaconda3/envs/elyra_ga/lib/python3.7/site-packages/tornado/escape.py", line 228, in to_unicode
raise TypeError("Expected bytes, unicode, or None; got %r" % type(value))
TypeError: Expected bytes, unicode, or None; got <class 'urllib3.exceptions.NewConnectionError'>

# Fixed code
def _upload_dependencies_to_object_store(
    self, runtime_configuration, pipeline_name, operation
):
    operation_artifact_archive = self._get_dependency_archive_name(operation)
    cos_directory = pipeline_name
    # upload operation dependencies to object store
    try:
        t0 = time.time()
        dependency_archive_path = self._generate_dependency_archive(operation)
        self.log_pipeline_info(
            pipeline_name,
            f"generated dependency archive: {dependency_archive_path}",
            operation_name=operation.name,
            duration=(time.time() - t0),
        )

        cos_client = CosClient(config=runtime_configuration)

        t0 = time.time()
        cos_client.upload_file_to_dir(
            dir=cos_directory,
            file_name=operation_artifact_archive,
            file_path=dependency_archive_path,
        )
        self.log_pipeline_info(
            pipeline_name,
            f"uploaded dependency archive to: {cos_directory}/{operation_artifact_archive}",
            operation_name=operation.name,
            duration=(time.time() - t0),
        )

    except FileNotFoundError as ex:
        self.log.error(
            "Dependencies were not found building archive for operation: {}".format(
                operation.name
            ),
            exc_info=True,
        )
        raise FileNotFoundError(
            "Node '{}' referenced dependencies that were not found: {}".format(
                operation.name, ex
            )
        ) from ex
    except MaxRetryError as ex:
        cos_endpoint = runtime_configuration.metadata.get("cos_endpoint")
        self.log.error(
            "Connection was refused when attempting to connect to : {}".format(
                cos_endpoint
            ),
            exc_info=True,
        )
        raise RuntimeError(
            "Connection was refused when attempting to upload artifacts to : '{}'. Please "
            "check your object storage settings. ".format(cos_endpoint)
        ) from ex
    except BaseException as ex:
        self.log.error(
            "Error uploading artifacts to object storage for operation: {}".format(
                operation.name
            ),
            exc_info=True,
        )
        raise ex from ex