# Buggy code
def _configure_iam_role(config):
    if "IamInstanceProfile" in config["head_node"]:
        return config

    profile = _get_instance_profile(DEFAULT_RAY_INSTANCE_PROFILE, config)

    if profile is None:
        logger.info(
            "Creating new instance profile {}".format(DEFAULT_RAY_INSTANCE_PROFILE)
        )
        client = _client("iam", config)
        client.create_instance_profile(InstanceProfileName=DEFAULT_RAY_INSTANCE_PROFILE)
        profile = _get_instance_profile(DEFAULT_RAY_INSTANCE_PROFILE, config)
        time.sleep(15)  # wait for propagation

    assert profile is not None, "Failed to create instance profile"

    if not profile.roles:
        role = _get_role(DEFAULT_RAY_IAM_ROLE, config)
        if role is None:
            logger.info("Creating new role {}".format(DEFAULT_RAY_IAM_ROLE))
            iam = _resource("iam", config)
            iam.create_role(
                RoleName=DEFAULT_RAY_IAM_ROLE,
                AssumeRolePolicyDocument=json.dumps(
                    {
                        "Statement": [
                            {
                                "Effect": "Allow",
                                "Principal": {"Service": "ec2.amazonaws.com"},
                                "Action": "sts:AssumeRole",
                            },
                        ],
                    }
                ),
            )
            role = _get_role(DEFAULT_RAY_IAM_ROLE, config)
            assert role is not None, "Failed to create role"
        role.attach_policy(PolicyArn="arn:aws:iam::aws:policy/AmazonEC2FullAccess")
        role.attach_policy(PolicyArn="arn:aws:iam::aws:policy/AmazonS3FullAccess")
        profile.add_role(RoleName=role.name)
        time.sleep(15)  # wait for propagation

    logger.info("Role not specified for head node, using {}".format(profile.arn))
    config["head_node"]["IamInstanceProfile"] = {"Arn": profile.arn}
    config["worker_nodes"]["IamInstanceProfile"] = {"Arn": profile.arn}

    return config


# Traceback (botocore.exceptions.ClientError)
ubuntu@ip-172-31-22-20:/tmp/ray/session_2018-11-01_22-01-45_16654/logs$ cat monitor.err
StandardAutoscaler: {'cluster_name': 'mastertest', 'min_workers': 200, 'max_workers': 200, 'docker': {'image': '', 'container_name': ''}, 'target_utilization_fraction': 0.8, 'idle_timeout_minutes': 5, 'provider': {'type': 'aws', 'region': 'us-west-2', 'availability_zone': 'us-west-2a,us-west-2b'}, 'auth': {'ssh_user': 'ubuntu', 'ssh_private_key': '~/ray_bootstrap_key.pem'}, 'head_node': {'InstanceType': 'm5.2xlarge', 'ImageId': 'ami-3b6bce43', 'BlockDeviceMappings': [{'DeviceName': '/dev/sda1', 'Ebs': {'VolumeSize': 50}}], 'IamInstanceProfile': {'Arn': 'arn:aws:iam::339530224232:instance-profile/ray-autoscaler-v1'}, 'KeyName': 'ray-autoscaler_us-west-2', 'SubnetIds': ['subnet-2154c944', 'subnet-4645f631'], 'SecurityGroupIds': ['sg-030b1764c1812e998']}, 'worker_nodes': {'InstanceType': 'm5.large', 'ImageId': 'ami-3b6bce43', 'InstanceMarketOptions': {'MarketType': 'spot'}, 'IamInstanceProfile': {'Arn': 'arn:aws:iam::339530224232:instance-profile/ray-autoscaler-v1'}, 'KeyName': 'ray-autoscaler_us-west-2', 'SubnetIds': ['subnet-2154c944', 'subnet-4645f631'], 'SecurityGroupIds': ['sg-030b1764c1812e998']}, 'file_mounts': {}, 'setup_commands': ['echo \'export PATH="$HOME/anaconda3/envs/tensorflow_p36/bin:$PATH"\' >> ~/.bashrc'], 'head_setup_commands': ['pip install boto3==1.4.8'], 'worker_setup_commands': [], 'head_start_ray_commands': ['ray stop', 'ulimit -n 65536; ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml'], 'worker_start_ray_commands': ['ray stop', 'ulimit -n 65536; ray start --redis-address=$RAY_HEAD_IP:6379 --object-manager-port=8076'], 'no_restart': False}
StandardAutoscaler [2018-11-01 22:01:46.799401]: 0/200 target nodes (0 pending)
- NodeIdleSeconds: Min=-1 Mean=-1 Max=-1
- NumNodesConnected: 0
- NumNodesUsed: 0.0
- ResourceUsage:
- TimeSinceLastHeartbeat: Min=-1 Mean=-1 Max=-1
StandardAutoscaler: Launching 5 new nodes
StandardAutoscaler [2018-11-01 22:01:46.830794]: 0/200 target nodes (5 pending)
- NodeIdleSeconds: Min=-1 Mean=-1 Max=-1
- NumNodesConnected: 0
- NumNodesUsed: 0.0
- ResourceUsage:
- TimeSinceLastHeartbeat: Min=-1 Mean=-1 Max=-1
Exception in thread Thread-1:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/autoscaler.py", line 251, in run
self._launch_node(config, count)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/autoscaler.py", line 242, in _launch_node
}, count)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/aws/node_provider.py", line 163, in create_node
self.ec2.create_instances(**conf)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/boto3/resources/factory.py", line 520, in do_action
response = action(self, *args, **kwargs)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/boto3/resources/action.py", line 83, in __call__
response = getattr(parent.meta.client, operation_name)(**params)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py", line 314, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py", line 612, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (UnauthorizedOperation) when calling the RunInstances operation: You are not authorized to perform this operation. Encoded authorization failure message: JUmMphVKvdVLB83WorEP2n5lOkl7LJT5E1K6lCJAxAjpvkJzdk--3vcnRXFh0Yj6Ez-XjrAda9hU472FJ_o2JIzby0EqMD2WD_qg7SqlgmahgDWBSwxOu4uCn_Py-OV1Cwj6XqFy6xJ9QcqsIfttWB9DstSHNQR_8y6SZ0-KYgUzzP51lLcYbTm2CK-D5mghExYd30aoyIV1YQpZ_8JyudvA8JhOFGVNrAIsYK_fT0iqsJOalAeTJAu-TUQNFxzUW6NENFT6xfN3bov6MPB2z0UvnFkMzH9fyerYzUXblO0qzdoEgyfxhcvhnq-7Dd6OJIBlycL5QF2XnR8czqmiSE3aQ09USKgKj1Oaru04EonLCRr64oVKMqpR80jTIGET7TTKDy-qqra-_uS2oQajd0T21V_y__GB7197KlSMi6JPSFHni7H6pZxOp3YTOneNBCydPrHCEmf6OFrbBtD7US-xIo_mW-LWMfHygRINgdAlPTQxBNfCWNpd7Mo9TK_02i0uNQaxR2Eb4sHQgPjWRRyN1gtsEA

StandardAutoscaler [2018-11-01 22:01:51.687415]: 0/200 target nodes (0 pending)
- NodeIdleSeconds: Min=4 Mean=4 Max=4
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 5 new nodes
StandardAutoscaler [2018-11-01 22:01:51.703377]: 0/200 target nodes (5 pending)
- NodeIdleSeconds: Min=4 Mean=4 Max=4
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Exception in thread Thread-2:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/autoscaler.py", line 251, in run
self._launch_node(config, count)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/autoscaler.py", line 242, in _launch_node
}, count)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/autoscaler/aws/node_provider.py", line 163, in create_node
self.ec2.create_instances(**conf)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/boto3/resources/factory.py", line 520, in do_action
response = action(self, *args, **kwargs)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/boto3/resources/action.py", line 83, in __call__
response = getattr(parent.meta.client, operation_name)(**params)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py", line 314, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py", line 612, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (UnauthorizedOperation) when calling the RunInstances operation: You are not authorized to perform this operation. Encoded authorization failure message: QOnTis6VbTYrt-07yg40sRrg8_SPtfkGI7lGq8R4bvzaISfzl48BMA5e6PeMiilTnDI40xDfLhJCVrCBIvvPoKljhdk-fssOX9yc174zfGLQRWZZGra1F8P3_bAEaAZn-KlWMZKbfh2XPeMw6U83Lbln2wuEWNaTxydbXyO4ADjz9PUsyXDijXopkp7VylGRBNcQy20718--eGZ5kIZlc5BJ40YTFU9JfaiMGLWTAV_hNcUrB5DOAtklZI4de4tYjxt88imIpp-slkiTbtJXcP4--bU1keOqIgbIP5vZMTrECvo95av4Tyq2BsydvK4PVviQSpqnZ9FlFUJi232jrxNYoH7-meIqGRsU9Xl715WOos2Yorgfqp5KyJn_aaxwWzwdPReN_aaIetpmQHJJtUj6c5R5jiJxtl9LxJNicT47JHvmPyJYb4iYNFHdAUMcrJryQCvZ6aNvz9IiG5ZVXyueSabABGskplcTZkpQ9wEib0kGbMyBncGR9kyAkmHrqOcauAjx1qpGawbrz16JXUED1FZJfQ

StandardAutoscaler [2018-11-01 22:01:56.744077]: 0/200 target nodes (0 pending)
- NodeIdleSeconds: Min=9 Mean=9 Max=9
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 5 new nodes
StandardAutoscaler [2018-11-01 22:01:56.767962]: 0/200 target nodes (5 pending)
- NodeIdleSeconds: Min=9 Mean=9 Max=9
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:01.814536]: 0/200 target nodes (5 pending)
- NodeIdleSeconds: Min=14 Mean=14 Max=14
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 5 new nodes
StandardAutoscaler [2018-11-01 22:02:01.841278]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=14 Mean=14 Max=14
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:06.875743]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=19 Mean=19 Max=19
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:06.893544]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=19 Mean=19 Max=19
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:11.929128]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=24 Mean=24 Max=24
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:11.945196]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=25 Mean=25 Max=25
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:16.983206]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=30 Mean=30 Max=30
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:16.997781]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=30 Mean=30 Max=30
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:22.051389]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=35 Mean=35 Max=35
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:22.067704]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=35 Mean=35 Max=35
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:27.109345]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=40 Mean=40 Max=40
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:27.125878]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=40 Mean=40 Max=40
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler [2018-11-01 22:02:32.157747]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=45 Mean=45 Max=45
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
StandardAutoscaler: Launching 0 new nodes
StandardAutoscaler [2018-11-01 22:02:32.172933]: 0/200 target nodes (10 pending)
- NodeIdleSeconds: Min=45 Mean=45 Max=45
- NumNodesConnected: 1
- NumNodesUsed: 0.0
- ResourceUsage: 0.0/8.0 b'CPU', 0.0/0.0 b'GPU'
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0

# Fixed code
def _configure_iam_role(config):
    if "IamInstanceProfile" in config["head_node"]:
        return config

    profile = _get_instance_profile(DEFAULT_RAY_INSTANCE_PROFILE, config)

    if profile is None:
        logger.info(
            "Creating new instance profile {}".format(DEFAULT_RAY_INSTANCE_PROFILE)
        )
        client = _client("iam", config)
        client.create_instance_profile(InstanceProfileName=DEFAULT_RAY_INSTANCE_PROFILE)
        profile = _get_instance_profile(DEFAULT_RAY_INSTANCE_PROFILE, config)
        time.sleep(15)  # wait for propagation

    assert profile is not None, "Failed to create instance profile"

    if not profile.roles:
        role = _get_role(DEFAULT_RAY_IAM_ROLE, config)
        if role is None:
            logger.info("Creating new role {}".format(DEFAULT_RAY_IAM_ROLE))
            iam = _resource("iam", config)
            iam.create_role(
                RoleName=DEFAULT_RAY_IAM_ROLE,
                AssumeRolePolicyDocument=json.dumps(
                    {
                        "Statement": [
                            {
                                "Effect": "Allow",
                                "Principal": {"Service": "ec2.amazonaws.com"},
                                "Action": "sts:AssumeRole",
                            },
                        ],
                    }
                ),
            )
            role = _get_role(DEFAULT_RAY_IAM_ROLE, config)
            assert role is not None, "Failed to create role"
        role.attach_policy(PolicyArn="arn:aws:iam::aws:policy/AmazonEC2FullAccess")
        role.attach_policy(PolicyArn="arn:aws:iam::aws:policy/AmazonS3FullAccess")
        profile.add_role(RoleName=role.name)
        time.sleep(15)  # wait for propagation

    logger.info("Role not specified for head node, using {}".format(profile.arn))
    config["head_node"]["IamInstanceProfile"] = {"Arn": profile.arn}

    return config