gurudesh/pytracebugs huggingface dataset:
# Buggy code
def set(self, key, value, state):
    raise NotImplementedError("Must implement the set method.")


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    raise NotImplementedError("Must implement the set method.")
# Buggy code
def _store_result(self, task_id, result, state, traceback=None, request=None, **kwargs):
    meta = self._get_result_meta(
        result=result, state=state, traceback=traceback, request=request
    )
    meta["task_id"] = bytes_to_str(task_id)

    # Retrieve metadata from the backend, if the status
    # is a success then we ignore any following update to the state.
    # This solves a task deduplication issue because of network
    # partitioning or lost workers. This issue involved a race condition
    # making a lost task overwrite the last successful result in the
    # result backend.
    current_meta = self._get_task_meta_for(task_id)

    if current_meta["status"] == states.SUCCESS:
        return result

    self.set(self.get_key_for_task(task_id), self.encode(meta), state)
    return result


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def _store_result(self, task_id, result, state, traceback=None, request=None, **kwargs):
    meta = self._get_result_meta(
        result=result, state=state, traceback=traceback, request=request
    )
    meta["task_id"] = bytes_to_str(task_id)

    # Retrieve metadata from the backend, if the status
    # is a success then we ignore any following update to the state.
    # This solves a task deduplication issue because of network
    # partitioning or lost workers. This issue involved a race condition
    # making a lost task overwrite the last successful result in the
    # result backend.
    current_meta = self._get_task_meta_for(task_id)

    if current_meta["status"] == states.SUCCESS:
        return result

    self._set_with_state(self.get_key_for_task(task_id), self.encode(meta), state)
    return result
# Buggy code
def _save_group(self, group_id, result):
    self.set(
        self.get_key_for_group(group_id),
        self.encode({"result": result.as_tuple()}),
        states.SUCCESS,
    )
    return result


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def _save_group(self, group_id, result):
    self._set_with_state(
        self.get_key_for_group(group_id),
        self.encode({"result": result.as_tuple()}),
        states.SUCCESS,
    )
    return result
# Buggy code
def set(self, key, value, state):
    return self.client.set(key, value, self.expires)


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    return self.client.set(key, value, self.expires)
# Buggy code
def set(self, key, value, state):
    """Set a key in Consul.

    Before creating the key it will create a session inside Consul
    where it creates a session with a TTL

    The key created afterwards will reference to the session's ID.

    If the session expires it will remove the key so that results
    can auto expire from the K/V store
    """
    session_name = bytes_to_str(key)

    key = self._key_to_consul_key(key)

    logger.debug(
        "Trying to create Consul session %s with TTL %d", session_name, self.expires
    )
    session_id = self.client.session.create(
        name=session_name, behavior="delete", ttl=self.expires
    )
    logger.debug("Created Consul session %s", session_id)

    logger.debug("Writing key %s to Consul", key)
    return self.client.kv.put(key=key, value=value, acquire=session_id)


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    """Set a key in Consul.

    Before creating the key it will create a session inside Consul
    where it creates a session with a TTL

    The key created afterwards will reference to the session's ID.

    If the session expires it will remove the key so that results
    can auto expire from the K/V store
    """
    session_name = bytes_to_str(key)

    key = self._key_to_consul_key(key)

    logger.debug(
        "Trying to create Consul session %s with TTL %d", session_name, self.expires
    )
    session_id = self.client.session.create(
        name=session_name, behavior="delete", ttl=self.expires
    )
    logger.debug("Created Consul session %s", session_id)

    logger.debug("Writing key %s to Consul", key)
    return self.client.kv.put(key=key, value=value, acquire=session_id)
# Buggy code
def set(self, key, value, state):
    """Store a value for a given key.

    Args:
          key: The key at which to store the value.
          value: The value to store.

    """
    key = bytes_to_str(key)
    LOGGER.debug(
        "Creating CosmosDB document %s/%s/%s",
        self._database_name,
        self._collection_name,
        key,
    )

    self._client.CreateDocument(
        self._collection_link, {"id": key, "value": value}, self._get_partition_key(key)
    )


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    """Store a value for a given key.

    Args:
          key: The key at which to store the value.
          value: The value to store.

    """
    key = bytes_to_str(key)
    LOGGER.debug(
        "Creating CosmosDB document %s/%s/%s",
        self._database_name,
        self._collection_name,
        key,
    )

    self._client.CreateDocument(
        self._collection_link, {"id": key, "value": value}, self._get_partition_key(key)
    )
# Buggy code
def set(self, key, value, state):
    self.connection.set(key, value, ttl=self.expires, format=FMT_AUTO)


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    self.connection.set(key, value, ttl=self.expires, format=FMT_AUTO)
# Buggy code
def set(self, key, value, state):
    key = bytes_to_str(key)
    data = {"_id": key, "value": value}
    try:
        self.connection.save(data)
    except pycouchdb.exceptions.Conflict:
        # document already exists, update it
        data = self.connection.get(key)
        data["value"] = value
        self.connection.save(data)


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    key = bytes_to_str(key)
    data = {"_id": key, "value": value}
    try:
        self.connection.save(data)
    except pycouchdb.exceptions.Conflict:
        # document already exists, update it
        data = self.connection.get(key)
        data["value"] = value
        self.connection.save(data)
# Buggy code
def set(self, key, value, state):
    key = string(key)
    request_parameters = self._prepare_put_request(key, value)
    self.client.put_item(**request_parameters)


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    key = string(key)
    request_parameters = self._prepare_put_request(key, value)
    self.client.put_item(**request_parameters)
# Buggy code
def set(self, key, value, state):
    body = {
        "result": value,
        "@timestamp": "{0}Z".format(datetime.utcnow().isoformat()[:-3]),
    }
    try:
        self._index(
            id=key,
            body=body,
        )
    except elasticsearch.exceptions.ConflictError:
        # document already exists, update it
        self._update(key, body, state)


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    return self._set_with_state(key, value, None)
# Buggy code
def set(self, key, value, state):
    with self.open(self._filename(key), "wb") as outfile:
        outfile.write(ensure_bytes(value))


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    with self.open(self._filename(key), "wb") as outfile:
        outfile.write(ensure_bytes(value))
# Buggy code
def set(self, key, value, state, **retry_policy):
    return self.ensure(self._set, (key, value), **retry_policy)


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value, **retry_policy):
    return self.ensure(self._set, (key, value), **retry_policy)
# Buggy code
def set(self, key, value, state):
    _key = self.bucket.new(key, data=value)
    _key.store()


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    _key = self.bucket.new(key, data=value)
    _key.store()
# Buggy code
def set(self, key, value, state):
    key = bytes_to_str(key)
    s3_object = self._get_s3_object(key)
    s3_object.put(Body=value)


# Traceback (TypeError)
celery_default_1      | [2020-06-04 19:51:47,723: ERROR/MainProcess] Pool callback raised exception: TypeError('set() takes 3 positional arguments but 4 were given',)
celery_default_1      | Traceback (most recent call last):
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/billiard/pool.py", line 1796, in safe_apply_callback
celery_default_1      |     fun(*args, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/worker/request.py", line 528, in on_failure
celery_default_1      |     store_result=self.store_errors,
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 169, in mark_as_failure
celery_default_1      |     traceback=traceback, request=request)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 443, in store_result
celery_default_1      |     request=request, **kwargs)
celery_default_1      |   File "/usr/share/dashboard/venv/lib/python3.6/site-packages/celery/backends/base.py", line 858, in _store_result
celery_default_1      |     self.set(self.get_key_for_task(task_id), self.encode(meta), state)
celery_default_1      | TypeError: set() takes 3 positional arguments but 4 were given

# Fixed code
def set(self, key, value):
    key = bytes_to_str(key)
    s3_object = self._get_s3_object(key)
    s3_object.put(Body=value)
# Buggy code
def unchain_tasks(self):
    # Clone chain's tasks assigning sugnatures from link_error
    # to each task
    tasks = [t.clone() for t in self.tasks]
    for sig in self.options.get("link_error", []):
        for task in tasks:
            task.link_error(sig)
    return tasks


# Traceback (RuntimeError)
Traceback (most recent call last):
File "test.py", line 30, in <module>
chain([chain(s2)]).apply_async() # issue
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 642, in apply_async
dict(self.options, **options) if options else self.options))
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 660, in run
task_id, group_id, chord,
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 721, in prepare_steps
task = task.clone(args, kwargs)
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 620, in clone
for sig in signature.kwargs['tasks']
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 1513, in maybe_signature
d = d.clone()
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 620, in clone
for sig in signature.kwargs['tasks']
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 1513, in maybe_signature
d = d.clone()
...
..
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 620, in clone
for sig in signature.kwargs['tasks']
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 1513, in maybe_signature
d = d.clone()
keeps repeating
..
..
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 617, in clone
signature = Signature.clone(self, *args, **kwargs)
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 272, in clone
app=self._app)
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 153, in from_dict
return target_cls.from_dict(d, app=app)
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 599, in from_dict
return _upgrade(d, _chain(tasks, app=app, **d['options']))
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 602, in __init__
tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0])
File "/bb/bin/dl/celery/4.4/kombu/utils/functional.py", line 256, in is_list
return isinstance(l, iters) and not isinstance(l, scalars or ())
File "/opt/bb/lib/python2.7/abc.py", line 132, in __instancecheck__
if subclass is not None and subclass in cls._abc_cache:
File "/opt/bb/lib/python2.7/_weakrefset.py", line 72, in __contains__
wr = ref(item)
RuntimeError: maximum recursion depth exceeded

# Fixed code
def unchain_tasks(self):
    # Clone chain's tasks assigning signatures from link_error
    # to each task
    tasks = [t.clone() for t in self.tasks]
    for sig in self.options.get("link_error", []):
        for task in tasks:
            task.link_error(sig)
    return tasks
# Buggy code
def __new__(cls, *tasks, **kwargs):
    # This forces `chain(X, Y, Z)` to work the same way as `X | Y | Z`
    if not kwargs and tasks:
        if len(tasks) != 1 or is_list(tasks[0]):
            tasks = tasks[0] if len(tasks) == 1 else tasks
            return reduce(operator.or_, tasks)
    return super(chain, cls).__new__(cls, *tasks, **kwargs)


# Traceback (RuntimeError)
Traceback (most recent call last):
File "test.py", line 30, in <module>
chain([chain(s2)]).apply_async() # issue
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 642, in apply_async
dict(self.options, **options) if options else self.options))
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 660, in run
task_id, group_id, chord,
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 721, in prepare_steps
task = task.clone(args, kwargs)
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 620, in clone
for sig in signature.kwargs['tasks']
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 1513, in maybe_signature
d = d.clone()
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 620, in clone
for sig in signature.kwargs['tasks']
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 1513, in maybe_signature
d = d.clone()
...
..
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 620, in clone
for sig in signature.kwargs['tasks']
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 1513, in maybe_signature
d = d.clone()
keeps repeating
..
..
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 617, in clone
signature = Signature.clone(self, *args, **kwargs)
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 272, in clone
app=self._app)
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 153, in from_dict
return target_cls.from_dict(d, app=app)
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 599, in from_dict
return _upgrade(d, _chain(tasks, app=app, **d['options']))
File "/bb/bin/dl/celery/4.4/celery/canvas.py", line 602, in __init__
tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0])
File "/bb/bin/dl/celery/4.4/kombu/utils/functional.py", line 256, in is_list
return isinstance(l, iters) and not isinstance(l, scalars or ())
File "/opt/bb/lib/python2.7/abc.py", line 132, in __instancecheck__
if subclass is not None and subclass in cls._abc_cache:
File "/opt/bb/lib/python2.7/_weakrefset.py", line 72, in __contains__
wr = ref(item)
RuntimeError: maximum recursion depth exceeded

# Fixed code
def __new__(cls, *tasks, **kwargs):
    # This forces `chain(X, Y, Z)` to work the same way as `X | Y | Z`
    if not kwargs and tasks:
        if len(tasks) != 1 or is_list(tasks[0]):
            tasks = tasks[0] if len(tasks) == 1 else tasks
            # if is_list(tasks) and len(tasks) == 1:
            #     return super(chain, cls).__new__(cls, tasks, **kwargs)
            return reduce(operator.or_, tasks, chain())
    return super(chain, cls).__new__(cls, *tasks, **kwargs)
# Buggy code
def __call__(self, *args, **kwargs):
    logger = get_logger(__name__)

    def handle_sigterm(signum, frame):
        logger.info("SIGTERM received, waiting till the task finished")

    signal.signal(signal.SIGTERM, handle_sigterm)
    _task_stack.push(self)
    self.push_request(args=args, kwargs=kwargs)
    try:
        return self.run(*args, **kwargs)
    finally:
        self.pop_request()
        _task_stack.pop()


# Traceback (ValueError)
Traceback (most recent call last):
File "/app/pipelines/serializers.py", line 184, in create
remote_worker=validated_data['with_remote'],
File "/usr/local/lib/python3.7/site-packages/celery/local.py", line 191, in __call__
return self._get_current_object()(*a, **kw)
File "/usr/local/lib/python3.7/site-packages/celery/app/task.py", line 396, in __call__
signal.signal(signal.SIGTERM, handle_sigterm)
File "/usr/local/lib/python3.7/signal.py", line 47, in signal
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
ValueError: signal only works in main thread

# Fixed code
def __call__(self, *args, **kwargs):
    _task_stack.push(self)
    self.push_request(args=args, kwargs=kwargs)
    try:
        return self.run(*args, **kwargs)
    finally:
        self.pop_request()
        _task_stack.pop()
# Buggy code
def remove_if_stale(self):
    """Remove the lock if the process isn't running.

    I.e. process does not respons to signal.
    """
    try:
        pid = self.read_pid()
    except ValueError:
        print("Broken pidfile found - Removing it.", file=sys.stderr)
        self.remove()
        return True
    if not pid:
        self.remove()
        return True

    try:
        os.kill(pid, 0)
    except os.error as exc:
        if exc.errno == errno.ESRCH:
            print("Stale pidfile exists - Removing it.", file=sys.stderr)
            self.remove()
            return True
    except SystemError:
        print("Stale pidfile exists - Removing it.", file=sys.stderr)
        self.remove()
        return True
    return False


# Traceback (OSError)
os.kill(2646, 0)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
OSError: [Errno 1] Operation not permitted

# Fixed code
def remove_if_stale(self):
    """Remove the lock if the process isn't running.

    I.e. process does not respons to signal.
    """
    try:
        pid = self.read_pid()
    except ValueError:
        print("Broken pidfile found - Removing it.", file=sys.stderr)
        self.remove()
        return True
    if not pid:
        self.remove()
        return True

    try:
        os.kill(pid, 0)
    except os.error as exc:
        if exc.errno == errno.ESRCH or exc.errno == errno.EPERM:
            print("Stale pidfile exists - Removing it.", file=sys.stderr)
            self.remove()
            return True
    except SystemError:
        print("Stale pidfile exists - Removing it.", file=sys.stderr)
        self.remove()
        return True
    return False
# Buggy code
def iter_native(self, result, no_ack=True, **kwargs):
    self._ensure_not_eager()

    results = result.results
    if not results:
        raise StopIteration()

    # we tell the result consumer to put consumed results
    # into these buckets.
    bucket = deque()
    for node in results:
        if node._cache:
            bucket.append(node)
        else:
            self._collect_into(node, bucket)

    for _ in self._wait_for_pending(result, no_ack=no_ack, **kwargs):
        while bucket:
            node = bucket.popleft()
            yield node.id, node._cache
    while bucket:
        node = bucket.popleft()
        yield node.id, node._cache


# Traceback (AttributeError)
Steps to Reproduce
Minimally Reproducible Test Case



from celery import  group, chain
from tasks import task as t

# failing sequence
task = group([ t.si(),t.si(), chain( t.si(), group([ t.si(), t.si()]))])

# working sequence
task = group([ t.si(),t.si(), chain( t.si(), group([ t.si(), t.si()]), t.si())])

async_result = task.apply_async()
result = async_result.get()



Expected Behavior
Calling get return group result.
Actual Behavior

All task finish success but calling .get() fail with traceback:
Traceback (most recent call last):
File "/my_app.py", line 111, in add_to_queue
result = async_result.get()
File "/root/.cache/pypoetry/virtualenvs/my_app/lib/python3.7/site-packages/celery/result.py", line 697, in get
on_interval=on_interval,
File "/root/.cache/pypoetry/virtualenvs/my_app/lib/python3.7/site-packages/celery/result.py", line 815, in join_native
on_message, on_interval):
File "/root/.cache/pypoetry/virtualenvs/my_app/lib/python3.7/site-packages/celery/backends/asynchronous.py", line 137, in iter_native
if node._cache:
AttributeError: 'GroupResult' object has no attribute '_cache'

# Fixed code
def iter_native(self, result, no_ack=True, **kwargs):
    self._ensure_not_eager()

    results = result.results
    if not results:
        raise StopIteration()

    # we tell the result consumer to put consumed results
    # into these buckets.
    bucket = deque()
    for node in results:
        if not hasattr(node, "_cache"):
            bucket.append(node)
        elif node._cache:
            bucket.append(node)
        else:
            self._collect_into(node, bucket)

    for _ in self._wait_for_pending(result, no_ack=no_ack, **kwargs):
        while bucket:
            node = bucket.popleft()
            if not hasattr(node, "_cache"):
                yield node.id, node.children
            else:
                yield node.id, node._cache
    while bucket:
        node = bucket.popleft()
        yield node.id, node._cache
# Buggy code
def __call__(self, *args, **kwargs):
    _task_stack.push(self)
    self.push_request(args=args, kwargs=kwargs)
    try:
        return self.run(*args, **kwargs)
    finally:
        self.pop_request()
        _task_stack.pop()


# Traceback (WorkerLostError)
Traceback (most recent call last):
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/billiard/pool.py", line 1171, in mark_as_worker_lost
human_status(exitcode)),
WorkerLostError: Worker exited prematurely: signal 15 (SIGTERM).

# Fixed code
def __call__(self, *args, **kwargs):
    logger = get_logger(__name__)
    handle_sigterm = lambda signum, frame: logger.info(
        "SIGTERM received, waiting till the task finished"
    )
    signal.signal(signal.SIGTERM, handle_sigterm)
    _task_stack.push(self)
    self.push_request(args=args, kwargs=kwargs)
    try:
        return self.run(*args, **kwargs)
    finally:
        self.pop_request()
        _task_stack.pop()
# Buggy code
def update_state(self, task_id=None, state=None, meta=None, **kwargs):
    """Update task state.

    Arguments:
        task_id (str): Id of the task to update.
            Defaults to the id of the current task.
        state (str): New state.
        meta (Dict): State meta-data.
    """
    if task_id is None:
        task_id = self.request.id
    self.backend.store_result(task_id, meta, state, **kwargs)


# Traceback (AttributeError)
Traceback (most recent call last):
worker_1_e608e69813d9 |   File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 385, in trace_task
worker_1_e608e69813d9 |     R = retval = fun(*args, **kwargs)
worker_1_e608e69813d9 |   File "/data/alaya/api/common/celery_app.py", line 113, in __call__
worker_1_e608e69813d9 |     return super(RequestContextTask, self).__call__(*args, **kwargs)
worker_1_e608e69813d9 |   File "/usr/local/lib/python2.7/dist-packages/celery/app/trace.py", line 648, in __protected_call__
worker_1_e608e69813d9 |     return self.run(*args, **kwargs)
worker_1_e608e69813d9 |   File "/data/alaya/api/scheduleoptimization/alns/core/alns.py", line 70, in do_something
worker_1_e608e69813d9 |     "some": "metadata",
worker_1_e608e69813d9 |   File "/data/alaya/api/common/celery_app.py", line 140, in update_state
worker_1_e608e69813d9 |     super(RequestContextTask, self).update_state(request=request, **args)
worker_1_e608e69813d9 |   File "/usr/local/lib/python2.7/dist-packages/celery/app/task.py", line 930, in update_state
worker_1_e608e69813d9 |     self.backend.store_result(task_id, meta, state, **kwargs)
worker_1_e608e69813d9 |   File "/data/alaya/api/common/multi_tenant_backend.py", line 51, in store_result
worker_1_e608e69813d9 |     tenant = request.get('_tenant')
worker_1_e608e69813d9 | AttributeError: 'NoneType' object has no attribute 'get'

# Fixed code
def update_state(self, task_id=None, state=None, meta=None, **kwargs):
    """Update task state.

    Arguments:
        task_id (str): Id of the task to update.
            Defaults to the id of the current task.
        state (str): New state.
        meta (Dict): State meta-data.
    """
    if task_id is None:
        task_id = self.request.id
    self.backend.store_result(task_id, meta, state, request=self.request, **kwargs)
# Buggy code
def link_error(self, sig):
    sig = sig.clone().set(immutable=True)
    return self.tasks[0].link_error(sig)


# Traceback (AttributeError)
Traceback (most recent call last):
File "eggs/celery-4.2.1-py2.7.egg/celery/app/trace.py", line 439, in trace_task
parent_id=uuid, root_id=root_id,
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 1232, in apply_async
return self.run(tasks, body, args, task_id=task_id, **options)
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 1277, in run
header_result = header(*partial_args, task_id=group_id, **options)
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 953, in __call__
return self.apply_async(partial_args, **options)
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 978, in apply_async
args=args, kwargs=kwargs, **options))
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 1054, in _apply_tasks
**options)
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 557, in apply_async
dict(self.options, **options) if options else self.options))
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 573, in run
task_id, group_id, chord,
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 683, in prepare_steps
task.link_error(errback)
File "eggs/celery-4.2.1-py2.7.egg/celery/canvas.py", line 1016, in link_error
sig = sig.clone().set(immutable=True)
AttributeError: 'dict' object has no attribute 'clone'

# Fixed code
def link_error(self, sig):
    try:
        sig = sig.clone().set(immutable=True)
    except AttributeError:
        # See issue #5265.  I don't use isinstance because current tests
        # pass a Mock object as argument.
        sig["immutable"] = True
        sig = Signature.from_dict(sig)
    return self.tasks[0].link_error(sig)
# Buggy code
def __init__(self, cert):
    assert crypto is not None
    with reraise_errors("Invalid certificate: {0!r}"):
        self._cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert)


# Traceback (kombu.exceptions.ContentDisallowed)
[2018-09-10 02:33:07,446: CRITICAL/MainProcess] Unrecoverable error: ContentDisallowed('Refusing to deserialize disabled content of type json (application/json)',)
Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/celery/worker/worker.py", line 205, in start
self.blueprint.start(self)
File "/usr/lib/python3.6/site-packages/celery/bootsteps.py", line 119, in start
step.start(parent)
File "/usr/lib/python3.6/site-packages/celery/bootsteps.py", line 369, in start
return self.obj.start()
File "/usr/lib/python3.6/site-packages/celery/worker/consumer/consumer.py", line 317, in start
blueprint.start(self)
File "/usr/lib/python3.6/site-packages/celery/bootsteps.py", line 119, in start
step.start(parent)
File "/usr/lib/python3.6/site-packages/celery/worker/consumer/consumer.py", line 593, in start
c.loop(*c.loop_args())
File "/usr/lib/python3.6/site-packages/celery/worker/loops.py", line 91, in asynloop
next(loop)
File "/usr/lib/python3.6/site-packages/kombu/asynchronous/hub.py", line 354, in create_loop
cb(*cbargs)
File "/usr/lib/python3.6/site-packages/kombu/transport/redis.py", line 1040, in on_readable
self.cycle.on_readable(fileno)
File "/usr/lib/python3.6/site-packages/kombu/transport/redis.py", line 337, in on_readable
chan.handlers[type]()
File "/usr/lib/python3.6/site-packages/kombu/transport/redis.py", line 667, in _receive
ret.append(self._receive_one(c))
File "/usr/lib/python3.6/site-packages/kombu/transport/redis.py", line 697, in _receive_one
message, self._fanout_to_queue[exchange])
File "/usr/lib/python3.6/site-packages/kombu/transport/virtual/base.py", line 983, in _deliver
callback(message)
File "/usr/lib/python3.6/site-packages/kombu/transport/virtual/base.py", line 633, in _callback
return callback(message)
File "/usr/lib/python3.6/site-packages/kombu/messaging.py", line 624, in _receive_callback
return on_m(message) if on_m else self.receive(decoded, message)
File "/usr/lib/python3.6/site-packages/celery/worker/consumer/gossip.py", line 201, in on_message
_, event = prepare(message.payload)
File "/usr/lib/python3.6/site-packages/kombu/message.py", line 207, in payload
return self._decoded_cache if self._decoded_cache else self.decode()
File "/usr/lib/python3.6/site-packages/kombu/message.py", line 192, in decode
self._decoded_cache = self._decode()
File "/usr/lib/python3.6/site-packages/kombu/message.py", line 197, in _decode
self.content_encoding, accept=self.accept)
File "/usr/lib/python3.6/site-packages/kombu/serialization.py", line 256, in loads
raise self._for_untrusted_content(content_type, 'disabled')
kombu.exceptions.ContentDisallowed: Refusing to deserialize disabled content of type json (application/json)

# Fixed code
def __init__(self, cert):
    with reraise_errors("Invalid certificate: {0!r}", errors=(ValueError,)):
        self._cert = load_pem_x509_certificate(
            ensure_bytes(cert), backend=default_backend()
        )