gurudesh/pytracebugs huggingface dataset:
# Buggy code
def __init__(self, host, port, redis_address, redis_password=None, log_dir=None):
    self.dashboard_head = dashboard_head.DashboardHead(
        http_host=host,
        http_port=port,
        redis_address=redis_address,
        redis_password=redis_password,
        log_dir=log_dir,
    )

    # Setup Dashboard Routes
    build_dir = setup_static_dir()
    logger.info("Setup static dir for dashboard: %s", build_dir)
    dashboard_utils.ClassMethodRouteTable.bind(self)


# Traceback (OSError)
2021-01-07 03:57:35,395	WARNING worker.py:1044 -- The dashboard on node travis-job-a2c3f054-a588-45a5-b10a-2456ba486c28 failed with the following error:
Traceback (most recent call last):
File "/opt/miniconda/lib/python3.6/asyncio/base_events.py", line 1073, in create_server
sock.bind(sa)
OSError: [Errno 98] Address already in use
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/travis/build/ray-project/ray/python/ray/new_dashboard/dashboard.py", line 197, in <module>
loop.run_until_complete(dashboard.run())
File "/opt/miniconda/lib/python3.6/asyncio/base_events.py", line 488, in run_until_complete
return future.result()
File "/home/travis/build/ray-project/ray/python/ray/new_dashboard/dashboard.py", line 100, in run
await self.dashboard_head.run()
File "/home/travis/build/ray-project/ray/python/ray/new_dashboard/head.py", line 220, in run
*(m.run(self.server) for m in modules))
File "/opt/miniconda/lib/python3.6/site-packages/aiohttp/web.py", line 369, in _run_app
await site.start()
File "/opt/miniconda/lib/python3.6/site-packages/aiohttp/web_runner.py", line 103, in start
reuse_port=self._reuse_port)
File "/opt/miniconda/lib/python3.6/asyncio/base_events.py", line 1077, in create_server
% (sa, err.strerror.lower()))
OSError: [Errno 98] error while attempting to bind on address ('127.0.0.1', 8265): address already in use

# Fixed code
def __init__(
    self, host, port, port_retries, redis_address, redis_password=None, log_dir=None
):
    self.dashboard_head = dashboard_head.DashboardHead(
        http_host=host,
        http_port=port,
        http_port_retries=port_retries,
        redis_address=redis_address,
        redis_password=redis_password,
        log_dir=log_dir,
    )

    # Setup Dashboard Routes
    try:
        build_dir = setup_static_dir()
        logger.info("Setup static dir for dashboard: %s", build_dir)
    except FrontendNotFoundError as ex:
        # Not to raise FrontendNotFoundError due to NPM incompatibilities
        # with Windows.
        # Please refer to ci.sh::build_dashboard_front_end()
        if sys.platform in ["win32", "cygwin"]:
            logger.warning(ex)
        else:
            raise ex
    dashboard_utils.ClassMethodRouteTable.bind(self)
# Buggy code
def _get(self, ref: ClientObjectRef, timeout: float):
    req = ray_client_pb2.GetRequest(id=ref.id, timeout=timeout)
    try:
        data = self.data_client.GetObject(req)
    except grpc.RpcError as e:
        raise e.details()
    if not data.valid:
        try:
            err = cloudpickle.loads(data.error)
        except Exception:
            logger.exception("Failed to deserialize {}".format(data.error))
            raise
        logger.error(err)
        raise err
    return loads_from_server(data.data)


# Traceback (ValueError)
Failed to deserialize b"\x80\x05\x95\xf1\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\nValueError\x94\x93\x94\x8c\xcfFailed to look up actor with name 'abc'. You are either trying to look up a named actor you didn't create, the named actor died, or the actor hasn't been created because named actor creation is asynchronous.\x94\x85\x94R\x94."
Traceback (most recent call last):
File "/Users/eoakes/code/ray/python/ray/util/client/worker.py", line 275, in _call_schedule_for_task
raise cloudpickle.loads(ticket.error)
ValueError: Failed to look up actor with name 'abc'. You are either trying to look up a named actor you didn't create, the named actor died, or the actor hasn't been created because named actor creation is asynchronous.

# Fixed code
def _get(self, ref: ClientObjectRef, timeout: float):
    req = ray_client_pb2.GetRequest(id=ref.id, timeout=timeout)
    try:
        data = self.data_client.GetObject(req)
    except grpc.RpcError as e:
        raise e.details()
    if not data.valid:
        try:
            err = cloudpickle.loads(data.error)
        except pickle.UnpicklingError:
            logger.exception("Failed to deserialize {}".format(data.error))
            raise
        logger.error(err)
        raise err
    return loads_from_server(data.data)
# Buggy code
def _call_schedule_for_task(self, task: ray_client_pb2.ClientTask) -> List[bytes]:
    logger.debug("Scheduling %s" % task)
    task.client_id = self._client_id
    try:
        ticket = self.server.Schedule(task, metadata=self.metadata)
    except grpc.RpcError as e:
        raise decode_exception(e.details)
    if not ticket.valid:
        try:
            raise cloudpickle.loads(ticket.error)
        except Exception:
            logger.exception("Failed to deserialize {}".format(ticket.error))
            raise
    return ticket.return_ids


# Traceback (ValueError)
Failed to deserialize b"\x80\x05\x95\xf1\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\nValueError\x94\x93\x94\x8c\xcfFailed to look up actor with name 'abc'. You are either trying to look up a named actor you didn't create, the named actor died, or the actor hasn't been created because named actor creation is asynchronous.\x94\x85\x94R\x94."
Traceback (most recent call last):
File "/Users/eoakes/code/ray/python/ray/util/client/worker.py", line 275, in _call_schedule_for_task
raise cloudpickle.loads(ticket.error)
ValueError: Failed to look up actor with name 'abc'. You are either trying to look up a named actor you didn't create, the named actor died, or the actor hasn't been created because named actor creation is asynchronous.

# Fixed code
def _call_schedule_for_task(self, task: ray_client_pb2.ClientTask) -> List[bytes]:
    logger.debug("Scheduling %s" % task)
    task.client_id = self._client_id
    try:
        ticket = self.server.Schedule(task, metadata=self.metadata)
    except grpc.RpcError as e:
        raise decode_exception(e.details)
    if not ticket.valid:
        try:
            raise cloudpickle.loads(ticket.error)
        except pickle.UnpicklingError:
            logger.exception("Failed to deserialize {}".format(ticket.error))
            raise
    return ticket.return_ids
# Buggy code
def normalize(data, wrt):
    """Normalize data to be in range (0,1), with respect to (wrt) boundaries,
    which can be specified.
    """
    return (data - np.min(wrt, axis=0)) / (np.max(wrt, axis=0) - np.min(wrt, axis=0))


# Traceback (ray.tune.error.TuneError)
Traceback (most recent call last):
File "./tune_pb2.py", line 303, in <module>
raise_on_failed_trial=False)
File "/home/john/anaconda3/envs/python3.7/lib/python3.7/site-packages/ray/tune/tune.py", line 411, in run
runner.step()
File "/home/john/anaconda3/envs/python3.7/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 572, in step
self.trial_executor.on_no_available_trials(self)
File "/home/john/anaconda3/envs/python3.7/lib/python3.7/site-packages/ray/tune/trial_executor.py", line 183, in on_no_available_trials
raise TuneError("There are paused trials, but no more pending "
ray.tune.error.TuneError: There are paused trials, but no more pending trials with sufficient resources.

# Fixed code
def normalize(data, wrt):
    """Normalize data to be in range (0,1), with respect to (wrt) boundaries,
    which can be specified.
    """
    return (data - np.min(wrt, axis=0)) / (
        np.max(wrt, axis=0) - np.min(wrt, axis=0) + 1e-8
    )
# Buggy code
def _check_ami(config):
    """Provide helpful message for missing ImageId for node configuration."""

    _set_config_info(head_ami_src="config", workers_ami_src="config")

    region = config["provider"]["region"]
    default_ami = DEFAULT_AMI.get(region)
    if not default_ami:
        # If we do not provide a default AMI for the given region, noop.
        return

    if config["head_node"].get("ImageId", "").lower() == "latest_dlami":
        config["head_node"]["ImageId"] = default_ami
        _set_config_info(head_ami_src="dlami")

    if config["worker_nodes"].get("ImageId", "").lower() == "latest_dlami":
        config["worker_nodes"]["ImageId"] = default_ami
        _set_config_info(workers_ami_src="dlami")


# Traceback (KeyError)
Traceback (most recent call last):
File "/home/ubuntu/.local/bin/ray", line 11, in <module>
load_entry_point('ray', 'console_scripts', 'ray')()
File "/home/ubuntu/ray/python/ray/scripts/scripts.py", line 1519, in main
return cli()
File "/home/ubuntu/.local/lib/python3.8/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/home/ubuntu/.local/lib/python3.8/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/home/ubuntu/.local/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/ubuntu/.local/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/ubuntu/.local/lib/python3.8/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/home/ubuntu/ray/python/ray/scripts/scripts.py", line 1171, in submit
create_or_update_cluster(
File "/home/ubuntu/ray/python/ray/autoscaler/_private/commands.py", line 226, in create_or_update_cluster
try_logging_config(config)
File "/home/ubuntu/ray/python/ray/autoscaler/_private/commands.py", line 73, in try_logging_config
log_to_cli(config)
File "/home/ubuntu/ray/python/ray/autoscaler/_private/aws/config.py", line 179, in log_to_cli
print_info(
File "/home/ubuntu/ray/python/ray/autoscaler/_private/aws/config.py", line 129, in print_info
head_value_str = config["head_node"][key]
KeyError: 'ImageId'

# Fixed code
def _check_ami(config):
    """Provide helpful message for missing ImageId for node configuration."""

    _set_config_info(head_ami_src="config", workers_ami_src="config")

    region = config["provider"]["region"]
    default_ami = DEFAULT_AMI.get(region)
    if not default_ami:
        # If we do not provide a default AMI for the given region, noop.
        return

    head_ami = config["head_node"].get("ImageId", "").lower()
    if head_ami in ["", "latest_dlami"]:
        config["head_node"]["ImageId"] = default_ami
        _set_config_info(head_ami_src="dlami")

    worker_ami = config["worker_nodes"].get("ImageId", "").lower()
    if worker_ami in ["", "latest_dlami"]:
        config["worker_nodes"]["ImageId"] = default_ami
        _set_config_info(workers_ami_src="dlami")
# Buggy code
def __init__(
    self, router: Router, endpoint_name, handle_options: Optional[HandleOptions] = None
):
    self.router = router
    self.endpoint_name = endpoint_name
    self.handle_options = handle_options or HandleOptions()


# Traceback (TypeError)
2021-01-04 21:15:59,360	INFO services.py:1173 -- View the Ray dashboard at http://127.0.0.1:8265
(pid=6369) 2021-01-04 21:16:01,432	INFO controller.py:346 -- Starting router with name 'hRhwaS:SERVE_CONTROLLER_ACTOR:SERVE_PROXY_ACTOR-node:192.168.31.141-0' on node 'node:192.168.31.141-0' listening on '127.0.0.1:8000'
(pid=6366) INFO:     Started server process [6366]
Traceback (most recent call last):
File "a.py", line 11, in <module>
ray.put(handle)
File "/Users/simonmo/miniconda3/envs/py37/lib/python3.7/site-packages/ray/worker.py", line 1411, in put
object_ref = worker.put_object(value, pin_object=True)
File "/Users/simonmo/miniconda3/envs/py37/lib/python3.7/site-packages/ray/worker.py", line 262, in put_object
serialized_value = self.get_serialization_context().serialize(value)
File "/Users/simonmo/miniconda3/envs/py37/lib/python3.7/site-packages/ray/serialization.py", line 406, in serialize
return self._serialize_to_msgpack(value)
File "/Users/simonmo/miniconda3/envs/py37/lib/python3.7/site-packages/ray/serialization.py", line 386, in _serialize_to_msgpack
self._serialize_to_pickle5(metadata, python_objects)
File "/Users/simonmo/miniconda3/envs/py37/lib/python3.7/site-packages/ray/serialization.py", line 346, in _serialize_to_pickle5
raise e
File "/Users/simonmo/miniconda3/envs/py37/lib/python3.7/site-packages/ray/serialization.py", line 343, in _serialize_to_pickle5
value, protocol=5, buffer_callback=writer.buffer_callback)
File "/Users/simonmo/miniconda3/envs/py37/lib/python3.7/site-packages/ray/cloudpickle/cloudpickle_fast.py", line 73, in dumps
cp.dump(obj)
File "/Users/simonmo/miniconda3/envs/py37/lib/python3.7/site-packages/ray/cloudpickle/cloudpickle_fast.py", line 563, in dump
return Pickler.dump(self, obj)
File "stringsource", line 2, in ray._raylet.Count.__reduce_cython__
TypeError: self.c_tag_keys,self.metric cannot be converted to a Python object for pickling
(pid=6369) 2021-01-04 21:16:03,924	INFO controller.py:753 -- Registering route 'f' to endpoint 'f' with methods '['GET']'.
2021-01-04 21:16:06,021	INFO api.py:65 -- Shutting down Ray Serve because client went out of scope. To prevent this, either keep a reference to the client object or use serve.start(detached=True).

# Fixed code
def __init__(
    self,
    router,  # ThreadProxiedRouter
    endpoint_name,
    handle_options: Optional[HandleOptions] = None,
):
    self.router = router
    self.endpoint_name = endpoint_name
    self.handle_options = handle_options or HandleOptions()
# Buggy code
def translate(
    configuration: Dict[str, Any], dictionary: Dict[str, str]
) -> Dict[str, Any]:
    return {dictionary[field]: configuration[field] for field in dictionary}


# Traceback (KeyError)
Traceback (most recent call last):
File "/home/ray/anaconda3/bin/ray-operator", line 8, in <module>
sys.exit(main())
File "/home/ray/anaconda3/lib/python3.7/site-packages/ray/operator/operator.py", line 123, in main
cluster_config = operator_utils.cr_to_config(cluster_cr)
File "/home/ray/anaconda3/lib/python3.7/site-packages/ray/operator/operator_utils.py", line 62, in cr_to_config
config["available_node_types" = get_node_types(cluster_resource)
File "/home/ray/anaconda3/lib/python3.7/site-packages/ray/operator/operator_utils.py", line 76, in get_node_types
pod_type_copy, dictionary=NODE_TYPE_FIELDS)
File "/home/ray/anaconda3/lib/python3.7/site-packages/ray/operator/operator_utils.py", line 98, in translate
return {dictionary[field: configuration[field for field in dictionary}
File "/home/ray/anaconda3/lib/python3.7/site-packages/ray/operator/operator_utils.py", line 98, in <dictcomp>
return {dictionary[field: configuration[field for field in dictionary}
KeyError: 'minWorkers'
stream closed

# Fixed code
def translate(
    configuration: Dict[str, Any], dictionary: Dict[str, str]
) -> Dict[str, Any]:
    return {
        dictionary[field]: configuration[field]
        for field in dictionary
        if field in configuration
    }
# Buggy code
def __init__(
    self, name: str, description: str = "", tag_keys: Optional[Tuple[str]] = None
):
    if len(name) == 0:
        raise ValueError("Empty name is not allowed. Please provide a metric name.")
    self._name = name
    self._description = description
    # We don't specify unit because it won't be
    # exported to Prometheus anyway.
    self._unit = ""
    # The default tags key-value pair.
    self._default_tags = {}
    # Keys of tags.
    self._tag_keys = tag_keys or tuple()
    # The Cython metric class. This should be set in the child class.
    self._metric = None

    if not isinstance(self._tag_keys, tuple):
        raise ValueError(
            f"tag_keys should be a tuple type, got: {type(self._tag_keys)}"
        )


# Traceback (AttributeError)
Traceback (most recent call last):
File "test.py", line 8, in <module>
count.record(1.0, {"b": 2})
File "/Users/eoakes/code/ray/python/ray/util/metrics.py", line 84, in record
self._metric.record(value, tags=final_tags)
File "python/ray/includes/metric.pxi", line 54, in ray._raylet.Metric.record
c_tags[tag_k.encode("ascii")] = tag_v.encode("ascii")
AttributeError: 'int' object has no attribute 'encode'

# Fixed code
def __init__(
    self, name: str, description: str = "", tag_keys: Optional[Tuple[str]] = None
):
    if len(name) == 0:
        raise ValueError("Empty name is not allowed. Please provide a metric name.")
    self._name = name
    self._description = description
    # We don't specify unit because it won't be
    # exported to Prometheus anyway.
    self._unit = ""
    # The default tags key-value pair.
    self._default_tags = {}
    # Keys of tags.
    self._tag_keys = tag_keys or tuple()
    # The Cython metric class. This should be set in the child class.
    self._metric = None

    if not isinstance(self._tag_keys, tuple):
        raise TypeError(f"tag_keys should be a tuple type, got: {type(self._tag_keys)}")

    for key in self._tag_keys:
        if not isinstance(key, str):
            raise TypeError(f"Tag keys must be str, got {type(key)}.")
# Buggy code
def set_default_tags(self, default_tags: Dict[str, str]):
    """Set default tags of metrics.

    Example:
        >>> # Note that set_default_tags returns the instance itself.
        >>> counter = Counter("name")
        >>> counter2 = counter.set_default_tags({"a": "b"})
        >>> assert counter is counter2
        >>> # this means you can instantiate it in this way.
        >>> counter = Counter("name").set_default_tags({"a": "b"})

    Args:
        default_tags(dict): Default tags that are
            used for every record method.

    Returns:
        Metric: it returns the instance itself.
    """
    self._default_tags = default_tags
    return self


# Traceback (AttributeError)
Traceback (most recent call last):
File "test.py", line 8, in <module>
count.record(1.0, {"b": 2})
File "/Users/eoakes/code/ray/python/ray/util/metrics.py", line 84, in record
self._metric.record(value, tags=final_tags)
File "python/ray/includes/metric.pxi", line 54, in ray._raylet.Metric.record
c_tags[tag_k.encode("ascii")] = tag_v.encode("ascii")
AttributeError: 'int' object has no attribute 'encode'

# Fixed code
def set_default_tags(self, default_tags: Dict[str, str]):
    """Set default tags of metrics.

    Example:
        >>> # Note that set_default_tags returns the instance itself.
        >>> counter = Counter("name")
        >>> counter2 = counter.set_default_tags({"a": "b"})
        >>> assert counter is counter2
        >>> # this means you can instantiate it in this way.
        >>> counter = Counter("name").set_default_tags({"a": "b"})

    Args:
        default_tags(dict): Default tags that are
            used for every record method.

    Returns:
        Metric: it returns the instance itself.
    """
    for key, val in default_tags.items():
        if key not in self._tag_keys:
            raise ValueError(f"Unrecognized tag key {key}.")
        if not isinstance(val, str):
            raise TypeError(f"Tag values must be str, got {type(val)}.")

    self._default_tags = default_tags
    return self
# Buggy code
def record(self, value: float, tags: dict = None) -> None:
    """Record the metric point of the metric.

    Args:
        value(float): The value to be recorded as a metric point.
    """
    assert self._metric is not None
    default_tag_copy = self._default_tags.copy()
    default_tag_copy.update(tags or {})
    self._metric.record(value, tags=default_tag_copy)


# Traceback (AttributeError)
Traceback (most recent call last):
File "test.py", line 8, in <module>
count.record(1.0, {"b": 2})
File "/Users/eoakes/code/ray/python/ray/util/metrics.py", line 84, in record
self._metric.record(value, tags=final_tags)
File "python/ray/includes/metric.pxi", line 54, in ray._raylet.Metric.record
c_tags[tag_k.encode("ascii")] = tag_v.encode("ascii")
AttributeError: 'int' object has no attribute 'encode'

# Fixed code
def record(self, value: float, tags: dict = None) -> None:
    """Record the metric point of the metric.

    Args:
        value(float): The value to be recorded as a metric point.
    """
    assert self._metric is not None
    if tags is not None:
        for val in tags.values():
            if not isinstance(val, str):
                raise TypeError(f"Tag values must be str, got {type(val)}.")

    default_tag_copy = self._default_tags.copy()
    default_tag_copy.update(tags or {})
    self._metric.record(value, tags=default_tag_copy)
# Buggy code
def get_cluster_info(self, type: ray_client_pb2.ClusterInfoType.TypeEnum):
    req = ray_client_pb2.ClusterInfoRequest()
    req.type = type
    resp = self.server.ClusterInfo(req, metadata=self.metadata)
    if resp.WhichOneof("response_type") == "resource_table":
        # translate from a proto map to a python dict
        output_dict = {k: v for k, v in resp.resource_table.table.items()}
        return output_dict
    return json.loads(resp.json)


# Traceback (AttributeError)
Traceback (most recent call last):
File "test.py", line 4, in <module>
print(ray.get_runtime_context().node_id.hex())
File "/Users/eoakes/code/ray/python/ray/runtime_context.py", line 58, in node_id
node_id = self.worker.current_node_id
File "/Users/eoakes/code/ray/python/ray/worker.py", line 148, in current_node_id
return self.core_worker.get_current_node_id()
AttributeError: 'Worker' object has no attribute 'core_worker'

# Fixed code
def get_cluster_info(self, type: ray_client_pb2.ClusterInfoType.TypeEnum):
    req = ray_client_pb2.ClusterInfoRequest()
    req.type = type
    resp = self.server.ClusterInfo(req, metadata=self.metadata)
    if resp.WhichOneof("response_type") == "resource_table":
        # translate from a proto map to a python dict
        output_dict = {k: v for k, v in resp.resource_table.table.items()}
        return output_dict
    elif resp.WhichOneof("response_type") == "runtime_context":
        return resp.runtime_context
    return json.loads(resp.json)
# Buggy code
def get(self):
    """Get a dictionary of the current_context.

    For fields that are not available (for example actor id inside a task)
    won't be included in the field.

    Returns:
        dict: Dictionary of the current context.
    """
    context = {
        "job_id": self.job_id,
        "node_id": self.node_id,
        "task_id": self.task_id,
        "actor_id": self.actor_id,
    }
    # Remove fields that are None.
    return {key: value for key, value in context.items() if value is not None}


# Traceback (AssertionError)
2021-01-13 14:15:29,011 INFO services.py:1169 -- View the Ray dashboard at http://127.0.0.1:8266
Traceback (most recent call last):
File "test.py", line 4, in <module>
print(ray.get_runtime_context().get())
File "/Users/eoakes/code/ray/python/ray/runtime_context.py", line 26, in get
"task_id": self.task_id,
File "/Users/eoakes/code/ray/python/ray/runtime_context.py", line 92, in task_id
assert self.worker.mode == ray.worker.WORKER_MODE, (
AssertionError: This method is only available when the process is a                 worker. Current mode: 0

# Fixed code
def get(self):
    """Get a dictionary of the current context.

    Fields that are not available (e.g., actor ID inside a task) won't be
    included in the field.

    Returns:
        dict: Dictionary of the current context.
    """
    context = {
        "job_id": self.job_id,
        "node_id": self.node_id,
    }
    if self.worker.mode == ray.worker.WORKER_MODE:
        if self.task_id is not None:
            context["task_id"] = self.task_id
        if self.actor_id is not None:
            context["actor_id"] = self.actor_id

    return context
# Buggy code
def __str__(self):
    return "The worker died unexpectedly while executing this task."


# Traceback (ray.exceptions.RayTaskError)
2020-10-05 01:55:09,393\u0009ERROR trial_runner.py:567 -- Trial PPO_QbertNoFrameskip-v4_b43b9_00027: Error processing event.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/ray/tune/trial_runner.py", line 515, in _process_trial
result = self.trial_executor.fetch_result(trial)
File "/usr/local/lib/python3.6/dist-packages/ray/tune/ray_trial_executor.py", line 488, in fetch_result
result = ray.get(trial_future[0], timeout=DEFAULT_GET_TIMEOUT)
File "/usr/local/lib/python3.6/dist-packages/ray/worker.py", line 1428, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError: \u001b[36mray::PPO.train()\u001b[39m (pid=4251, ip=172.30.96.106)
File "python/ray/_raylet.pyx", line 484, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 438, in ray._raylet.execute_task.function_executor
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer.py", line 516, in train
raise e
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer.py", line 505, in train
result = Trainable.train(self)
File "/usr/local/lib/python3.6/dist-packages/ray/tune/trainable.py", line 336, in train
result = self.step()
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer_template.py", line 134, in step
res = next(self.train_exec_impl)
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 756, in __next__
return next(self.built_iterator)
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 783, in apply_foreach
for item in it:
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 791, in apply_foreach
result = fn(item)
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/execution/metric_ops.py", line 79, in __call__
timeout_seconds=self.timeout_seconds)
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/evaluation/metrics.py", line 75, in collect_episodes
metric_lists = ray.get(collected)
ray.exceptions.RayActorError: The actor died unexpectedly before finishing this task.

# Fixed code
def __str__(self):
    return (
        "The worker died unexpectedly while executing this task. "
        "Check python-core-worker-*.log files for more information."
    )
# Buggy code
def __str__(self):
    return "The actor died unexpectedly before finishing this task."


# Traceback (ray.exceptions.RayTaskError)
2020-10-05 01:55:09,393\u0009ERROR trial_runner.py:567 -- Trial PPO_QbertNoFrameskip-v4_b43b9_00027: Error processing event.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/ray/tune/trial_runner.py", line 515, in _process_trial
result = self.trial_executor.fetch_result(trial)
File "/usr/local/lib/python3.6/dist-packages/ray/tune/ray_trial_executor.py", line 488, in fetch_result
result = ray.get(trial_future[0], timeout=DEFAULT_GET_TIMEOUT)
File "/usr/local/lib/python3.6/dist-packages/ray/worker.py", line 1428, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError: \u001b[36mray::PPO.train()\u001b[39m (pid=4251, ip=172.30.96.106)
File "python/ray/_raylet.pyx", line 484, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 438, in ray._raylet.execute_task.function_executor
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer.py", line 516, in train
raise e
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer.py", line 505, in train
result = Trainable.train(self)
File "/usr/local/lib/python3.6/dist-packages/ray/tune/trainable.py", line 336, in train
result = self.step()
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer_template.py", line 134, in step
res = next(self.train_exec_impl)
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 756, in __next__
return next(self.built_iterator)
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 783, in apply_foreach
for item in it:
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 791, in apply_foreach
result = fn(item)
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/execution/metric_ops.py", line 79, in __call__
timeout_seconds=self.timeout_seconds)
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/evaluation/metrics.py", line 75, in collect_episodes
metric_lists = ray.get(collected)
ray.exceptions.RayActorError: The actor died unexpectedly before finishing this task.

# Fixed code
def __str__(self):
    return (
        "The actor died unexpectedly before finishing this task. "
        "Check python-core-worker-*.log files for more information."
    )
# Buggy code
def _random_string():
    id_hash = hashlib.sha1()
    id_hash.update(uuid.uuid4().bytes)
    id_bytes = id_hash.digest()
    assert len(id_bytes) == ray_constants.ID_SIZE
    return id_bytes


# Traceback (ray.exceptions.RayTaskError)
2020-10-05 01:55:09,393\u0009ERROR trial_runner.py:567 -- Trial PPO_QbertNoFrameskip-v4_b43b9_00027: Error processing event.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/ray/tune/trial_runner.py", line 515, in _process_trial
result = self.trial_executor.fetch_result(trial)
File "/usr/local/lib/python3.6/dist-packages/ray/tune/ray_trial_executor.py", line 488, in fetch_result
result = ray.get(trial_future[0], timeout=DEFAULT_GET_TIMEOUT)
File "/usr/local/lib/python3.6/dist-packages/ray/worker.py", line 1428, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError: \u001b[36mray::PPO.train()\u001b[39m (pid=4251, ip=172.30.96.106)
File "python/ray/_raylet.pyx", line 484, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 438, in ray._raylet.execute_task.function_executor
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer.py", line 516, in train
raise e
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer.py", line 505, in train
result = Trainable.train(self)
File "/usr/local/lib/python3.6/dist-packages/ray/tune/trainable.py", line 336, in train
result = self.step()
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/agents/trainer_template.py", line 134, in step
res = next(self.train_exec_impl)
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 756, in __next__
return next(self.built_iterator)
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 783, in apply_foreach
for item in it:
File "/usr/local/lib/python3.6/dist-packages/ray/util/iter.py", line 791, in apply_foreach
result = fn(item)
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/execution/metric_ops.py", line 79, in __call__
timeout_seconds=self.timeout_seconds)
File "/usr/local/lib/python3.6/dist-packages/ray/rllib/evaluation/metrics.py", line 75, in collect_episodes
metric_lists = ray.get(collected)
ray.exceptions.RayActorError: The actor died unexpectedly before finishing this task.

# Fixed code
def _random_string():
    id_hash = hashlib.shake_128()
    id_hash.update(uuid.uuid4().bytes)
    id_bytes = id_hash.digest(ray_constants.ID_SIZE)
    assert len(id_bytes) == ray_constants.ID_SIZE
    return id_bytes
# Buggy code
def shutdown(self) -> None:
    """Completely shut down the connected Serve instance.

    Shuts down all processes and deletes all state associated with the
    instance.
    """
    if not self._shutdown:
        ray.get(self._controller.shutdown.remote())
        ray.kill(self._controller, no_restart=True)
        self._shutdown = True


# Traceback (ray.serve.exceptions.RayServeException)
File descriptor limit 256 is too low for production servers and may result in connection errors. At least 8192 is recommended. --- Fix with 'ulimit -n 8192'
2020-11-20 10:38:27,065	INFO services.py:1173 -- View the Ray dashboard at http://127.0.0.1:8265
(pid=raylet) 2020-11-20 10:38:29,628	INFO controller.py:313 -- Starting router with name 'SERVE_CONTROLLER_ACTOR:SERVE_PROXY_ACTOR-node:192.168.31.141-0' on node 'node:192.168.31.141-0' listening on '127.0.0.1:8000'
(pid=32847) INFO:     Started server process [32847]
Traceback (most recent call last):
File "detached.py", line 5, in <module>
client = serve.start(detached=True)
File "/Users/simonmo/Desktop/ray/ray/python/ray/serve/api.py", line 414, in start
raise RayServeException("Called serve.start(detached=True) but a "
ray.serve.exceptions.RayServeException: Called serve.start(detached=True) but a detached instance is already running. Please use serve.connect() to connect to the running instance instead.

# Fixed code
def shutdown(self) -> None:
    """Completely shut down the connected Serve instance.

    Shuts down all processes and deletes all state associated with the
    instance.
    """
    if (not self._shutdown) and ray.is_initialized():
        ray.get(self._controller.shutdown.remote())
        ray.kill(self._controller, no_restart=True)

        # Wait for the named actor entry gets removed as well.
        started = time.time()
        while True:
            try:
                ray.get_actor(self._controller_name)
                if time.time() - started > 5:
                    logger.warning(
                        "Waited 5s for Serve to shutdown gracefully but "
                        "the controller is still not cleaned up. "
                        "You can ignore this warning if you are shutting "
                        "down the Ray cluster."
                    )
                    break
            except ValueError:  # actor name is removed
                break

        self._shutdown = True
# Buggy code
def custom_excepthook(type, value, tb):
    # If this is a driver, push the exception to GCS worker table.
    if global_worker.mode == SCRIPT_MODE:
        error_message = "".join(traceback.format_tb(tb))
        worker_id = global_worker.worker_id
        worker_type = ray.gcs_utils.DRIVER
        worker_info = {"exception": error_message}

        ray.state.state.add_worker(worker_id, worker_type, worker_info)
    # Call the normal excepthook.
    normal_excepthook(type, value, tb)


# Traceback (AttributeError)
raise Exception("hello")
Error in sys.excepthook:
Traceback (most recent call last):
File "/Users/eoakes/code/ray/python/ray/worker.py", line 836, in custom_excepthook
ray.state.state.add_worker(worker_id, worker_type, worker_info)
File "/Users/eoakes/code/ray/python/ray/state.py", line 733, in add_worker
return self.global_state_accessor.add_worker_info(
AttributeError: 'NoneType' object has no attribute 'add_worker_info'

Original exception was:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
Exception: hello

# Fixed code
def custom_excepthook(type, value, tb):
    # If this is a driver, push the exception to GCS worker table.
    if global_worker.mode == SCRIPT_MODE:
        error_message = "".join(traceback.format_tb(tb))
        worker_id = global_worker.worker_id
        worker_type = ray.gcs_utils.DRIVER
        worker_info = {"exception": error_message}

        ray.state.state._check_connected()
        ray.state.state.add_worker(worker_id, worker_type, worker_info)
    # Call the normal excepthook.
    normal_excepthook(type, value, tb)
# Buggy code
async def _do_long_poll(self):
    while True:
        updates: Dict[str, UpdatedObject] = await self._poll_once()
        self._update(updates)
        logger.debug(f"LongPollerClient received updates: {updates}")
        for key, updated_object in updates.items():
            # NOTE(simon): This blocks the loop from doing another poll.
            # Consider use loop.create_task here or poll first then call
            # the callbacks.
            callback = self.key_listeners[key]
            await callback(updated_object.object_snapshot)


# Traceback (AttributeError)
Exception in callback async_set_result.<locals>.set_future()
handle: <Handle async_set_result.<locals>.set_future()>
Traceback (most recent call last):
File "/Users/simonmo/miniconda3/lib/python3.6/asyncio/events.py", line 145, in _run
self._callback(*self._args)
File "python/ray/_raylet.pyx", line 1530, in ray._raylet.async_set_result.set_future
AttributeError: 'RayActorError' object has no attribute 'as_instanceof_cause'

# Fixed code
async def _do_long_poll(self):
    while True:
        try:
            updates: Dict[str, UpdatedObject] = await self._poll_once()
            self._update(updates)
            logger.debug(f"LongPollerClient received udpates: {updates}")
            for key, updated_object in updates.items():
                # NOTE(simon):
                # This blocks the loop from doing another poll. Consider
                # use loop.create_task here or poll first then call the
                # callbacks.
                callback = self.key_listeners[key]
                await callback(updated_object.object_snapshot)
        except ray.exceptions.RayActorError:
            # This can happen during shutdown where the controller is
            # intentionally killed, the client should just gracefully
            # exit.
            logger.debug("LongPollerClient failed to connect to host. Shutting down.")
            break
# Buggy code
def __init__(self):
    self.indent_level = 0

    self._verbosity = 0
    self._color_mode = "auto"
    self._log_style = "record"
    self.pretty = False
    self.interactive = False

    # store whatever colorful has detected for future use if
    # the color ouput is toggled (colorful detects # of supported colors,
    # so it has some non-trivial logic to determine this)
    self._autodetected_cf_colormode = cf.colorful.colormode
    self.set_format()


# Traceback (ray.tune.error.TuneError)
2020-11-19 19:42:27,243 VINFO updater.py:460 -- `rsync`ed /root/ray_results/tune_mnist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (remote) to /root/ray_results/tune_mn
ist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (local)
2020-11-19 19:42:27,247 ERROR trial_runner.py:868 -- Trial DEFAULT_1d092_00006: Error handling checkpoint /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 864, in _process_trial_save
trial.on_checkpoint(trial.saving_to)
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial.py", line 498, in on_checkpoint
self, checkpoint.value))
ray.tune.error.TuneError: Trial DEFAULT_1d092_00006: Checkpoint path /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/ not found after successful sync down.
2020-11-19 19:42:27,250 WARNING util.py:140 -- The `process_trial_save` operation took 1.05049467086792 seconds to complete, which may be a performance bottleneck.

# Fixed code
def __init__(self):
    self.indent_level = 0

    self._verbosity = 0
    self._verbosity_overriden = False
    self._color_mode = "auto"
    self._log_style = "record"
    self.pretty = False
    self.interactive = False

    # store whatever colorful has detected for future use if
    # the color ouput is toggled (colorful detects # of supported colors,
    # so it has some non-trivial logic to determine this)
    self._autodetected_cf_colormode = cf.colorful.colormode
    self.set_format()
# Buggy code
def verbosity(self):
    if not self.pretty:
        return 999
    return self._verbosity


# Traceback (ray.tune.error.TuneError)
2020-11-19 19:42:27,243 VINFO updater.py:460 -- `rsync`ed /root/ray_results/tune_mnist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (remote) to /root/ray_results/tune_mn
ist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (local)
2020-11-19 19:42:27,247 ERROR trial_runner.py:868 -- Trial DEFAULT_1d092_00006: Error handling checkpoint /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 864, in _process_trial_save
trial.on_checkpoint(trial.saving_to)
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial.py", line 498, in on_checkpoint
self, checkpoint.value))
ray.tune.error.TuneError: Trial DEFAULT_1d092_00006: Checkpoint path /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/ not found after successful sync down.
2020-11-19 19:42:27,250 WARNING util.py:140 -- The `process_trial_save` operation took 1.05049467086792 seconds to complete, which may be a performance bottleneck.

# Fixed code
def verbosity(self):
    if self._verbosity_overriden:
        return self._verbosity
    elif not self.pretty:
        return 999
    return self._verbosity
# Buggy code
def _set_verbosity(self, x):
    self._verbosity = x


# Traceback (ray.tune.error.TuneError)
2020-11-19 19:42:27,243 VINFO updater.py:460 -- `rsync`ed /root/ray_results/tune_mnist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (remote) to /root/ray_results/tune_mn
ist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (local)
2020-11-19 19:42:27,247 ERROR trial_runner.py:868 -- Trial DEFAULT_1d092_00006: Error handling checkpoint /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 864, in _process_trial_save
trial.on_checkpoint(trial.saving_to)
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial.py", line 498, in on_checkpoint
self, checkpoint.value))
ray.tune.error.TuneError: Trial DEFAULT_1d092_00006: Checkpoint path /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/ not found after successful sync down.
2020-11-19 19:42:27,250 WARNING util.py:140 -- The `process_trial_save` operation took 1.05049467086792 seconds to complete, which may be a performance bottleneck.

# Fixed code
def _set_verbosity(self, x):
    self._verbosity = x
    self._verbosity_overriden = True
# Buggy code
def __init__(
    self, local_dir: str, remote_dir: str, sync_client: Optional[SyncClient] = None
):
    self.local_ip = services.get_node_ip_address()
    self.worker_ip = None

    sync_client = sync_client or DockerSyncClient()
    sync_client.configure(self._cluster_config_file)

    super(NodeSyncer, self).__init__(local_dir, remote_dir, sync_client)


# Traceback (ray.tune.error.TuneError)
2020-11-19 19:42:27,243 VINFO updater.py:460 -- `rsync`ed /root/ray_results/tune_mnist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (remote) to /root/ray_results/tune_mn
ist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (local)
2020-11-19 19:42:27,247 ERROR trial_runner.py:868 -- Trial DEFAULT_1d092_00006: Error handling checkpoint /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 864, in _process_trial_save
trial.on_checkpoint(trial.saving_to)
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial.py", line 498, in on_checkpoint
self, checkpoint.value))
ray.tune.error.TuneError: Trial DEFAULT_1d092_00006: Checkpoint path /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/ not found after successful sync down.
2020-11-19 19:42:27,250 WARNING util.py:140 -- The `process_trial_save` operation took 1.05049467086792 seconds to complete, which may be a performance bottleneck.

# Fixed code
def __init__(
    self, local_dir: str, remote_dir: str, sync_client: Optional[SyncClient] = None
):
    configure_logging(
        log_style="record", verbosity=env_integer("TUNE_SYNCER_VERBOSITY", 0)
    )
    self.local_ip = services.get_node_ip_address()
    self.worker_ip = None

    sync_client = sync_client or DockerSyncClient()
    sync_client.configure(self._cluster_config_file)

    super(NodeSyncer, self).__init__(local_dir, remote_dir, sync_client)
# Buggy code
def unflatten_dict(dt, delimiter="/"):
    """Unflatten dict. Does not support unflattening lists."""
    out = defaultdict(dict)
    for key, val in dt.items():
        path = key.split(delimiter)
        item = out
        for k in path[:-1]:
            item = item[k]
        item[path[-1]] = val
    return dict(out)


# Traceback (KeyError)
$ python ./python/ray/tune/examples/bohb_example.py
File descriptor limit 256 is too low for production servers and may result in connection errors. At least 8192 is recommended. --- Fix with 'ulimit -n 8192'
2020-11-11 20:38:44,944 INFO services.py:1110 -- View the Ray dashboard at http://127.0.0.1:8265
Traceback (most recent call last):
File "./python/ray/tune/examples/bohb_example.py", line 87, in <module>
stop={"training_iteration": 100})
File "/Users/hartikainen/github/ray-project/ray/python/ray/tune/tune.py", line 416, in run
runner.step()
File "/Users/hartikainen/github/ray-project/ray/python/ray/tune/trial_runner.py", line 380, in step
next_trial = self._get_next_trial()  # blocking
File "/Users/hartikainen/github/ray-project/ray/python/ray/tune/trial_runner.py", line 479, in _get_next_trial
self._update_trial_queue(blocking=wait_for_trial)
File "/Users/hartikainen/github/ray-project/ray/python/ray/tune/trial_runner.py", line 855, in _update_trial_queue
trial = self._search_alg.next_trial()
File "/Users/hartikainen/github/ray-project/ray/python/ray/tune/suggest/search_generator.py", line 114, in next_trial
self._experiment.dir_name)
File "/Users/hartikainen/github/ray-project/ray/python/ray/tune/suggest/search_generator.py", line 121, in create_trial_if_possible
suggested_config = self.searcher.suggest(trial_id)
File "/Users/hartikainen/github/ray-project/ray/python/ray/tune/suggest/bohb.py", line 175, in suggest
return unflatten_dict(config)
File "/Users/hartikainen/github/ray-project/ray/python/ray/tune/utils/util.py", line 276, in unflatten_dict
item = item[k]
KeyError: 'b'

# Fixed code
def unflatten_dict(dt, delimiter="/"):
    """Unflatten dict. Does not support unflattening lists."""
    dict_type = type(dt)
    out = dict_type()
    for key, val in dt.items():
        path = key.split(delimiter)
        item = out
        for k in path[:-1]:
            item = item.setdefault(k, dict_type())
        item[path[-1]] = val
    return out