# Buggy code
def _configure_iam_role(config):
    """Setup a gcp service account with IAM roles.

    Creates a gcp service acconut and binds IAM roles which allow it to control
    control storage/compute services. Specifically, the head node needs to have
    an IAM role that allows it to create further gce instances and store items
    in google cloud storage.

    TODO: Allow the name/id of the service account to be configured
    """
    email = SERVICE_ACCOUNT_EMAIL_TEMPLATE.format(
        account_id=DEFAULT_SERVICE_ACCOUNT_ID,
        project_id=config["provider"]["project_id"],
    )
    service_account = _get_service_account(email, config)

    if service_account is None:
        logger.info(
            "Creating new service account {}".format(DEFAULT_SERVICE_ACCOUNT_ID)
        )

        service_account = _create_service_account(
            DEFAULT_SERVICE_ACCOUNT_ID, DEFAULT_SERVICE_ACCOUNT_CONFIG, config
        )

    assert service_account is not None, "Failed to create service account"

    _add_iam_policy_binding(service_account, DEFAULT_SERVICE_ACCOUNT_ROLES)

    # NOTE: The amount of access is determined by the scope + IAM
    # role of the service account. Even if the cloud-platform scope
    # gives (scope) access to the whole cloud-platform, the service
    # account is limited by the IAM rights specified below.
    config["head_node"]["serviceAccounts"] = [
        {
            "email": service_account["email"],
            "scopes": ["https://www.googleapis.com/auth/cloud-platform"],
        }
    ]
    config["worker_nodes"]["serviceAccounts"] = [
        {
            "email": service_account["email"],
            "scopes": ["https://www.googleapis.com/auth/cloud-platform"],
        }
    ]

    return config


# Traceback (botocore.exceptions.ClientError)
ubuntu@ip-172-31-22-20:/tmp/ray/session_2018-11-01_22-01-45_16654/logs$ cat monitor.err
StandardAutoscaler: {'cluster_name': 'mastertest', 'min_workers': 200, 'max_workers': 200, 'docker': {'image': '', 'container_name': ''}, 'target_utilization_fraction': 0.8, 'idle_timeout_minutes': 5, 'provider': {'type': 'aws', 'region': 'us-west-2', 'availability_zone': 'us-west-2a,us-west-2b'}, 'auth': {'ssh_user': 'ubuntu', 'ssh_private_key': '~/ray_bootstrap_key.pem'}, 'head_node': {'InstanceType': 'm5.2xlarge', 'ImageId': 'ami-3b6bce43', 'BlockDeviceMappings': [{'DeviceName': '/dev/sda1', 'Ebs': {'VolumeSize': 50}}], 'IamInstanceProfile': {'Arn': 'arn:aws:iam::339530224232:instance-profile/ray-autoscaler-v1'}, 'KeyName': 'ray-autoscaler_us-west-2', 'SubnetIds': ['subnet-2154c944', 'subnet-4645f631'], 'SecurityGroupIds': ['sg-030b1764c1812e998']}, 'worker_nodes': {'InstanceType': 'm5.large', 'ImageId': 'ami-3b6bce43', 'InstanceMarketOptions': {'MarketType': 'spot'}, 'IamInstanceProfile': {'Arn': 'arn:aws:iam::339530224232:instance-profile/ray-autoscaler-v1'}, 'KeyName': 'ray-autoscaler_us-west-2', 'SubnetIds': ['subnet-2154c944', 'subnet-4645f631'], 'SecurityGroupIds': ['sg-030b1764c1812e998']}, 'file_mounts': {}, 'setup_commands': ['echo \'export PATH="$HOME/anaconda3/envs/tensorflow_p36/bin:$PATH"\' >> ~/.bashrc'], 'head_setup_commands': ['pip install boto3==1.4.8'], 'worker_setup_commands': [], 'head_start_ray_commands': ['ray stop', 'ulimit -n 65536; ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml'], 'worker_start_ray_commands': ['ray stop', 'ulimit -n 65536; ray start --redis-address=$RAY_HEAD_IP:6379 --object-manager-port=8076'], 'no_restart': False}
StandardAutoscaler [2018-11-01 22:01:46.799401]: 0/200 target nodes (0 pending)
- NodeIdleSeconds: Min=-1 Mean=-1 Max=-1
- NumNodesConnected: 0
- NumNodesUsed: 0.0
- ResourceUsage:
- TimeSinceLastHeartbeat: Min=-1 Mean=-1 Max=-1
StandardAutoscaler: Launching 5 new nodes
StandardAutoscaler [2018-11-01 22:01:46.830794]: 0/200 target nodes (5 pending)
- NodeIdleSeconds: Min=-1 Mean=-1 Max=-1
- NumNodesConnected: 0
- NumNodesUsed: 0.0
- ResourceUsage:
- TimeSinceLastHeartbeat: Min=-1 Mean=-1 Max=-1
Exception in thread Thread-1:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/autoscaler.py", line 251, in run
self._launch_node(config, count)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/autoscaler.py", line 242, in _launch_node
}, count)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/aws/node_provider.py", line 163, in create_node
self.ec2.create_instances(**conf)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/boto3/resources/factory.py", line 520, in do_action
response = action(self, *args, **kwargs)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/boto3/resources/action.py", line 83, in __call__
response = getattr(parent.meta.client, operation_name)(**params)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py", line 314, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py", line 612, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (UnauthorizedOperation) when calling the RunInstances operation: You are not authorized to perform this operation. Encoded authorization failure message: JUmMphVKvdVLB83WorEP2n5lOkl7LJT5E1K6lCJAxAjpvkJzdk--3vcnRXFh0Yj6Ez-XjrAda9hU472FJ_o2JIzby0EqMD2WD_qg7SqlgmahgDWBSwxOu4uCn_Py-OV1Cwj6XqFy6xJ9QcqsIfttWB9DstSHNQR_8y6SZ0-KYgUzzP51lLcYbTm2CK-D5mghExYd30aoyIV1YQpZ_8JyudvA8JhOFGVNrAIsYK_fT0iqsJOalAeTJAu-TUQNFxzUW6NENFT6xfN3bov6MPB2z0UvnFkMzH9fyerYzUXblO0qzdoEgyfxhcvhnq-7Dd6OJIBlycL5QF2XnR8czqmiSE3aQ09USKgKj1Oaru04EonLCRr64oVKMqpR80jTIGET7TTKDy-qqra-_uS2oQajd0T21V_y__GB7197KlSMi6JPSFHni7H6pZxOp3YTOneNBCydPrHCEmf6OFrbBtD7US-xIo_mW-LWMfHygRINgdAlPTQxBNfCWNpd7Mo9TK_02i0uNQaxR2Eb4sHQgPjWRRyN1gtsEA

StandardAutoscaler [2018-11-01 22:01:51.687415]: 0/200 target nodes (0 pending)
- NodeIdleSeconds: Min=4 Mean=4 Max=4
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 5 new nodes
StandardAutoscaler [2018-11-01 22:01:51.703377]: 0/200 target nodes (5 pending)
- NodeIdleSeconds: Min=4 Mean=4 Max=4
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Exception in thread Thread-2:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/autoscaler.py", line 251, in run
self._launch_node(config, count)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/autoscaler.py", line 242, in _launch_node
}, count)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/aws/node_provider.py", line 163, in create_node
self.ec2.create_instances(**conf)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/boto3/resources/factory.py", line 520, in do_action
response = action(self, *args, **kwargs)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/boto3/resources/action.py", line 83, in __call__
response = getattr(parent.meta.client, operation_name)(**params)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py", line 314, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py", line 612, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (UnauthorizedOperation) when calling the RunInstances operation: You are not authorized to perform this operation. Encoded authorization failure message: QOnTis6VbTYrt-07yg40sRrg8_SPtfkGI7lGq8R4bvzaISfzl48BMA5e6PeMiilTnDI40xDfLhJCVrCBIvvPoKljhdk-fssOX9yc174zfGLQRWZZGra1F8P3_bAEaAZn-KlWMZKbfh2XPeMw6U83Lbln2wuEWNaTxydbXyO4ADjz9PUsyXDijXopkp7VylGRBNcQy20718--eGZ5kIZlc5BJ40YTFU9JfaiMGLWTAV_hNcUrB5DOAtklZI4de4tYjxt88imIpp-slkiTbtJXcP4--bU1keOqIgbIP5vZMTrECvo95av4Tyq2BsydvK4PVviQSpqnZ9FlFUJi232jrxNYoH7-meIqGRsU9Xl715WOos2Yorgfqp5KyJn_aaxwWzwdPReN_aaIetpmQHJJtUj6c5R5jiJxtl9LxJNicT47JHvmPyJYb4iYNFHdAUMcrJryQCvZ6aNvz9IiG5ZVXyueSabABGskplcTZkpQ9wEib0kGbMyBncGR9kyAkmHrqOcauAjx1qpGawbrz16JXUED1FZJfQ

StandardAutoscaler [2018-11-01 22:01:56.744077]: 0/200 target nodes (0 pending)
- NodeIdleSeconds: Min=9 Mean=9 Max=9
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 5 new nodes
StandardAutoscaler [2018-11-01 22:01:56.767962]: 0/200 target nodes (5 pending)
- NodeIdleSeconds: Min=9 Mean=9 Max=9
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:01.814536]: 0/200 target nodes (5 pending)
- NodeIdleSeconds: Min=14 Mean=14 Max=14
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 5 new nodes
StandardAutoscaler [2018-11-01 22:02:01.841278]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=14 Mean=14 Max=14
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:06.875743]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=19 Mean=19 Max=19
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:06.893544]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=19 Mean=19 Max=19
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:11.929128]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=24 Mean=24 Max=24
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:11.945196]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=25 Mean=25 Max=25
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:16.983206]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=30 Mean=30 Max=30
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:16.997781]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=30 Mean=30 Max=30
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:22.051389]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=35 Mean=35 Max=35
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:22.067704]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=35 Mean=35 Max=35
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:27.109345]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=40 Mean=40 Max=40
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:27.125878]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=40 Mean=40 Max=40
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:32.157747]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=45 Mean=45 Max=45
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:32.172933]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=45 Mean=45 Max=45
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0

# Fixed code
def _configure_iam_role(config):
    """Setup a gcp service account with IAM roles.

    Creates a gcp service acconut and binds IAM roles which allow it to control
    control storage/compute services. Specifically, the head node needs to have
    an IAM role that allows it to create further gce instances and store items
    in google cloud storage.

    TODO: Allow the name/id of the service account to be configured
    """
    email = SERVICE_ACCOUNT_EMAIL_TEMPLATE.format(
        account_id=DEFAULT_SERVICE_ACCOUNT_ID,
        project_id=config["provider"]["project_id"],
    )
    service_account = _get_service_account(email, config)

    if service_account is None:
        logger.info(
            "Creating new service account {}".format(DEFAULT_SERVICE_ACCOUNT_ID)
        )

        service_account = _create_service_account(
            DEFAULT_SERVICE_ACCOUNT_ID, DEFAULT_SERVICE_ACCOUNT_CONFIG, config
        )

    assert service_account is not None, "Failed to create service account"

    _add_iam_policy_binding(service_account, DEFAULT_SERVICE_ACCOUNT_ROLES)

    config["head_node"]["serviceAccounts"] = [
        {
            "email": service_account["email"],
            # NOTE: The amount of access is determined by the scope + IAM
            # role of the service account. Even if the cloud-platform scope
            # gives (scope) access to the whole cloud-platform, the service
            # account is limited by the IAM rights specified below.
            "scopes": ["https://www.googleapis.com/auth/cloud-platform"],
        }
    ]

    return config