gurudesh/pytracebugs huggingface dataset:
# Buggy code
def on_recurring(self, reference_doc, subscription_doc):
    mcount = month_map[subscription_doc.frequency]
    self.set(
        "delivery_date",
        get_next_date(
            reference_doc.delivery_date, mcount, cint(subscription_doc.repeat_on_day)
        ),
    )

    for d in self.get("items"):
        reference_delivery_date = frappe.db.get_value(
            "Sales Order Item",
            {"parent": reference_doc.name, "item_code": d.item_code, "idx": d.idx},
            "delivery_date",
        )

        d.set(
            "delivery_date",
            get_next_date(
                reference_delivery_date, mcount, cint(subscription_doc.repeat_on_day)
            ),
        )


# Traceback (AttributeError)
Traceback (most recent call last):
File "/Users/makarandbauskar/workspace/frappe/dev-bench/apps/erpnext/erpnext/subscription/doctype/subscription/subscription.py", line 120, in create_documents
send_notification(doc, data.print_format, data.recipients)
File "/Users/makarandbauskar/workspace/frappe/dev-bench/apps/erpnext/erpnext/subscription/doctype/subscription/subscription.py", line 178, in send_notification
print_format = print_format or new_rv.recurring_print_format
AttributeError: 'DeliveryNote' object has no attribute 'recurring_print_format'

# Fixed code
def on_recurring(self, reference_doc, subscription_doc):
    self.set(
        "delivery_date",
        get_next_schedule_date(
            reference_doc.delivery_date,
            subscription_doc.frequency,
            cint(subscription_doc.repeat_on_day),
        ),
    )

    for d in self.get("items"):
        reference_delivery_date = frappe.db.get_value(
            "Sales Order Item",
            {"parent": reference_doc.name, "item_code": d.item_code, "idx": d.idx},
            "delivery_date",
        )

        d.set(
            "delivery_date",
            get_next_schedule_date(
                reference_delivery_date,
                subscription_doc.frequency,
                cint(subscription_doc.repeat_on_day),
            ),
        )
# Buggy code
def get_events(start, end, filters=None):
    events = []

    employee = frappe.db.get_value(
        "Employee", {"user_id": frappe.session.user}, ["name", "company"], as_dict=True
    )
    if employee:
        employee, company = employee.name, employee.company
    else:
        employee = ""
        company = frappe.db.get_value("Global Defaults", None, "default_company")

    from frappe.desk.reportview import get_filters_cond

    conditions = get_filters_cond("Leave Application")

    # show department leaves for employee
    if "Employee" in frappe.get_roles():
        add_department_leaves(events, start, end, employee, company)

    add_leaves(events, start, end, conditions)

    add_block_dates(events, start, end, employee, company)
    add_holidays(events, start, end, employee, company)

    return events


# Traceback (TypeError)
Traceback (most recent call last):
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/app.py", line 56, in application
response = frappe.handler.handle()
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/handler.py", line 21, in handle
data = execute_cmd(cmd)
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/handler.py", line 52, in execute_cmd
return frappe.call(method, **frappe.form_dict)
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/__init__.py", line 922, in call
return fn(*args, **newargs)
File "/home/frappe/benches/bench-2017-07-18/apps/erpnext/erpnext/hr/doctype/leave_application/leave_application.py", line 415, in get_events
conditions = get_filters_cond("Leave Application")
TypeError: get_filters_cond() takes at least 3 arguments (1 given)

# Fixed code
def get_events(start, end, filters=None):
    events = []

    employee = frappe.db.get_value(
        "Employee", {"user_id": frappe.session.user}, ["name", "company"], as_dict=True
    )
    if employee:
        employee, company = employee.name, employee.company
    else:
        employee = ""
        company = frappe.db.get_value("Global Defaults", None, "default_company")

    from frappe.desk.reportview import get_filters_cond

    conditions = get_filters_cond("Leave Application", filters, [])

    # show department leaves for employee
    if "Employee" in frappe.get_roles():
        add_department_leaves(events, start, end, employee, company)

    add_leaves(events, start, end, conditions)

    add_block_dates(events, start, end, employee, company)
    add_holidays(events, start, end, employee, company)

    return events
# Buggy code
def add_department_leaves(events, start, end, employee, company):
    department = frappe.db.get_value("Employee", employee, "department")

    if not department:
        return

    # department leaves
    department_employees = frappe.db.sql_list(
        """select name from tabEmployee where department=%s
		and company=%s""",
        (department, company),
    )

    match_conditions = 'employee in ("%s")' % '", "'.join(department_employees)
    add_leaves(events, start, end, match_conditions=match_conditions)


# Traceback (TypeError)
Traceback (most recent call last):
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/app.py", line 56, in application
response = frappe.handler.handle()
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/handler.py", line 21, in handle
data = execute_cmd(cmd)
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/handler.py", line 52, in execute_cmd
return frappe.call(method, **frappe.form_dict)
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/__init__.py", line 922, in call
return fn(*args, **newargs)
File "/home/frappe/benches/bench-2017-07-18/apps/erpnext/erpnext/hr/doctype/leave_application/leave_application.py", line 415, in get_events
conditions = get_filters_cond("Leave Application")
TypeError: get_filters_cond() takes at least 3 arguments (1 given)

# Fixed code
def add_department_leaves(events, start, end, employee, company):
    department = frappe.db.get_value("Employee", employee, "department")

    if not department:
        return

    # department leaves
    department_employees = frappe.db.sql_list(
        """select name from tabEmployee where department=%s
		and company=%s""",
        (department, company),
    )

    match_conditions = 'and employee in ("%s")' % '", "'.join(department_employees)
    add_leaves(events, start, end, match_conditions=match_conditions)
# Buggy code
def add_leaves(events, start, end, match_conditions=None):
    query = """select name, from_date, to_date, employee_name, half_day,
		status, employee, docstatus
		from `tabLeave Application` where
		from_date <= %(end)s and to_date >= %(start)s <= to_date
		and docstatus < 2
		and status!="Rejected" """
    if match_conditions:
        query += " and " + match_conditions

    for d in frappe.db.sql(query, {"start": start, "end": end}, as_dict=True):
        e = {
            "name": d.name,
            "doctype": "Leave Application",
            "from_date": d.from_date,
            "to_date": d.to_date,
            "status": d.status,
            "title": cstr(d.employee_name) + (d.half_day and _(" (Half Day)") or ""),
            "docstatus": d.docstatus,
        }
        if e not in events:
            events.append(e)


# Traceback (TypeError)
Traceback (most recent call last):
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/app.py", line 56, in application
response = frappe.handler.handle()
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/handler.py", line 21, in handle
data = execute_cmd(cmd)
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/handler.py", line 52, in execute_cmd
return frappe.call(method, **frappe.form_dict)
File "/home/frappe/benches/bench-2017-07-18/apps/frappe/frappe/__init__.py", line 922, in call
return fn(*args, **newargs)
File "/home/frappe/benches/bench-2017-07-18/apps/erpnext/erpnext/hr/doctype/leave_application/leave_application.py", line 415, in get_events
conditions = get_filters_cond("Leave Application")
TypeError: get_filters_cond() takes at least 3 arguments (1 given)

# Fixed code
def add_leaves(events, start, end, match_conditions=None):
    query = """select name, from_date, to_date, employee_name, half_day,
		status, employee, docstatus
		from `tabLeave Application` where
		from_date <= %(end)s and to_date >= %(start)s <= to_date
		and docstatus < 2
		and status!="Rejected" """
    if match_conditions:
        query += match_conditions

    for d in frappe.db.sql(query, {"start": start, "end": end}, as_dict=True):
        e = {
            "name": d.name,
            "doctype": "Leave Application",
            "from_date": d.from_date,
            "to_date": d.to_date,
            "status": d.status,
            "title": cstr(d.employee_name) + (d.half_day and _(" (Half Day)") or ""),
            "docstatus": d.docstatus,
        }
        if e not in events:
            events.append(e)
# Buggy code
def __init__(self, key: str, value: str, reason: Any = None):
    message = "invalid value {value!r} ({type}) for key {key!r}{reason}".format(
        value=value, type=type(value), key=key, reason=": " + reason if reason else ""
    )
    super().__init__(message)


# Traceback (intelmq.lib.exceptions.PipelineError)
abusix-expert-cz: Bot has found a problem.
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/bot.py", line 269, in start
self.process()
File "/usr/local/lib/python3.8/dist-packages/intelmq/bots/experts/abusix/expert.py", line 26, in process
event = self.receive_message()
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/bot.py", line 606, in receive_message
self.__current_message = libmessage.MessageFactory.unserialize(message,
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/message.py", line 72, in unserialize
return MessageFactory.from_dict(message, harmonization=harmonization,
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/message.py", line 54, in from_dict
return class_reference(message, auto=True, harmonization=harmonization)
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/message.py", line 539, in __init__
super().__init__(template, auto, harmonization)
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/message.py", line 127, in __init__
self.add(key, value, sanitize=True)
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/message.py", line 249, in add
raise exceptions.InvalidValue(key, old_value)
intelmq.lib.exceptions.InvalidValue: invalid value 0 (<class 'int'>) for key 'source.asn'
abusix-expert-cz: Current Message(event): None.
abusix-expert-cz: Bot will continue in 15 seconds.
abusix-expert-cz: Pipeline failed.
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/bot.py", line 269, in start
self.process()
File "/usr/local/lib/python3.8/dist-packages/intelmq/bots/experts/abusix/expert.py", line 26, in process
event = self.receive_message()
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/usr/local/lib/python3.8/dist-packages/intelmq/lib/pipeline.py", line 122, in receive
raise exceptions.PipelineError("There's already a message, first "
intelmq.lib.exceptions.PipelineError: pipeline failed - "There's already a message, first acknowledge the existing one."
abusix-expert-cz: Bot will continue in 15 seconds.

# Fixed code
def __init__(self, key: str, value: str, reason: Any = None, object: bytes = None):
    message = "invalid value {value!r} ({type}) for key {key!r}{reason}".format(
        value=value, type=type(value), key=key, reason=": " + reason if reason else ""
    )
    self.object = object
    super().__init__(message)
# Buggy code
def is_valid(value: str, sanitize: bool = False) -> bool:
    if sanitize:
        value = IPAddress().sanitize(value)

    if not GenericType().is_valid(value):
        return False

    try:
        address = ipaddress.ip_address(value)
    except ValueError:
        return False

    if address == ipaddress.ip_address("0.0.0.0"):
        return False

    return True


# Traceback (AssertionError)
[  168s] ======================================================================
[  168s] FAIL: test_ipaddress_valid_invalid (intelmq.tests.lib.test_harmonization.TestHarmonization)
[  168s] Test IPAddress.is_valid with invalid arguments.
[  168s] ----------------------------------------------------------------------
[  168s] Traceback (most recent call last):
[  168s]   File "/home/abuild/rpmbuild/BUILD/intelmq/intelmq/tests/lib/test_harmonization.py", line 123, in test_ipaddress_valid_invalid
[  168s]     self.assertFalse(harmonization.IPAddress.is_valid('fe80::1c41:b16d:ff5e:689d%bnep0',
[  168s] AssertionError: True is not false

# Fixed code
def is_valid(value: str, sanitize: bool = False) -> bool:
    if sanitize:
        value = IPAddress().sanitize(value)

    if not GenericType().is_valid(value):
        return False

    try:
        address = ipaddress.ip_address(value)
    except ValueError:
        return False

    if address == ipaddress.ip_address("0.0.0.0"):
        return False

    if "%" in address:
        # IPv6 address with scope ID
        # https://github.com/certtools/intelmq/issues/1550
        return False

    return True
# Buggy code
def __init__(self, argument):
    if type(argument) is type and issubclass(argument, Exception):
        message = "pipeline failed - %s" % traceback.format_exc(argument)
    else:
        message = "pipeline failed - %s" % repr(argument)
    super().__init__(message)


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def __init__(self, argument: Exception):
    if type(argument) is type and issubclass(argument, Exception):
        message = "pipeline failed - %s" % traceback.format_exc(argument)
    else:
        message = "pipeline failed - %s" % repr(argument)
    super().__init__(message)
# Buggy code
def __init__(self, key):
    message = "key %s not exists" % repr(key)
    super().__init__(message)


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def __init__(self, encodings=None, exception: UnicodeDecodeError = None, object=None):
    self.object = object
    suffix = []
    if encodings:
        suffix.append("with given encodings %r" % encodings)
    if exception:
        suffix.append(
            "at position %s with length %d (%r)"
            ""
            % (
                exception.start,
                exception.end,
                exception.object[exception.start : exception.end],
            )
        )
        suffix.append("with reason %r" % exception.reason)
    suffix = (" " + " ".join(suffix)) if suffix else ""
    super().__init__("Could not decode string%s." % suffix)
# Buggy code
def receive(self) -> str:
    if self._has_message:
        raise exceptions.PipelineError(
            "There's already a message, first acknowledge the existing one."
        )

    retval = self._receive()
    self._has_message = True
    return retval


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def receive(self) -> str:
    if self._has_message:
        raise exceptions.PipelineError(
            "There's already a message, first acknowledge the existing one."
        )

    retval = self._receive()
    self._has_message = True
    return utils.decode(retval)
# Buggy code
def _receive(self) -> str:
    raise NotImplementedError


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def _receive(self) -> bytes:
    raise NotImplementedError
# Buggy code
def acknowledge(self):
    if not self._has_message:
        raise exceptions.PipelineError("No message to acknowledge.")
    self._acknowledge()
    self._has_message = False


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def acknowledge(self):
    """
    Acknowledge/delete the current message from the source queue

    Parameters
    ----------

    Raises
    ------
    exceptions
        exceptions.PipelineError: If no message is held

    Returns
    -------
    None.

    """
    if not self._has_message:
        raise exceptions.PipelineError("No message to acknowledge.")
    self._acknowledge()
    self._has_message = False
# Buggy code
def _receive(self) -> str:
    if self.source_queue is None:
        raise exceptions.ConfigurationError("pipeline", "No source queue given.")
    try:
        while True:
            try:
                retval = self.pipe.lindex(
                    self.internal_queue, -1
                )  # returns None if no value
            except (
                redis.exceptions.BusyLoadingError
            ):  # Just wait at redis' startup #1334
                time.sleep(1)
            else:
                break
        if not retval:
            retval = self.pipe.brpoplpush(self.source_queue, self.internal_queue, 0)
        return utils.decode(retval)
    except Exception as exc:
        raise exceptions.PipelineError(exc)


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def _receive(self) -> bytes:
    if self.source_queue is None:
        raise exceptions.ConfigurationError("pipeline", "No source queue given.")
    try:
        while True:
            try:
                retval = self.pipe.lindex(
                    self.internal_queue, -1
                )  # returns None if no value
            except (
                redis.exceptions.BusyLoadingError
            ):  # Just wait at redis' startup #1334
                time.sleep(1)
            else:
                break
        if not retval:
            retval = self.pipe.brpoplpush(self.source_queue, self.internal_queue, 0)
    except Exception as exc:
        raise exceptions.PipelineError(exc)
    else:
        return retval
# Buggy code
def _receive(self) -> str:
    """
    Receives the last not yet acknowledged message.

    Does not block unlike the other pipelines.
    """
    if len(self.state.get(self.internal_queue, [])) > 0:
        return utils.decode(self.state[self.internal_queue].pop(0))

    first_msg = self.state[self.source_queue].pop(0)

    if self.internal_queue in self.state:
        self.state[self.internal_queue].append(first_msg)
    else:
        self.state[self.internal_queue] = [first_msg]

    return utils.decode(first_msg)


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def _receive(self) -> bytes:
    """
    Receives the last not yet acknowledged message.

    Does not block unlike the other pipelines.
    """
    if len(self.state[self.internal_queue]) > 0:
        return utils.decode(self.state[self.internal_queue][0])

    try:
        first_msg = self.state[self.source_queue].pop(0)
    except IndexError as exc:
        raise exceptions.PipelineError(exc)
    self.state[self.internal_queue].append(first_msg)

    return first_msg
# Buggy code
def _receive(self) -> str:
    if self.source_queue is None:
        raise exceptions.ConfigurationError("pipeline", "No source queue given.")
    try:
        method, header, body = next(self.channel.consume(self.source_queue))
        if method:
            self.delivery_tag = method.delivery_tag
            return utils.decode(body)
    except Exception as exc:
        raise exceptions.PipelineError(exc)


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def _receive(self) -> bytes:
    if self.source_queue is None:
        raise exceptions.ConfigurationError("pipeline", "No source queue given.")
    try:
        method, header, body = next(self.channel.consume(self.source_queue))
        if method:
            self.delivery_tag = method.delivery_tag
    except Exception as exc:
        raise exceptions.PipelineError(exc)
    else:
        return body
# Buggy code
def decode(
    text: Union[bytes, str], encodings: Sequence[str] = ("utf-8",), force: bool = False
) -> str:
    """
    Decode given string to UTF-8 (default).

    Parameters:
        text: if unicode string is given, same object is returned
        encodings: list/tuple of encodings to use
        force: Ignore invalid characters

    Returns:
        converted unicode string

    Raises:
        ValueError: if decoding failed
    """
    if isinstance(text, str):
        return text

    for encoding in encodings:
        try:
            return str(text.decode(encoding))
        except ValueError:
            pass

    if force:
        for encoding in encodings:
            try:
                return str(text.decode(encoding, "ignore"))
            except ValueError:
                pass

    raise ValueError(
        "Could not decode string with given encodings{!r}.".format(encodings)
    )


# Traceback (ValueError)
2020-01-24 12:07:28,164 - bot-id - ERROR - Pipeline failed.
Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/pipeline.py", line 249, in _receive
return utils.decode(retval)
File "/.../intelmq/intelmq/lib/utils.py", line 102, in decode
".".format(encodings))
ValueError: Could not decode string with given encodings('utf-8',).

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/.../intelmq/intelmq/lib/bot.py", line 271, in start
self.process()
File "/.../intelmq/intelmq/bots/experts/sieve/expert.py", line 89, in process
event = self.receive_message()
File "/.../intelmq/intelmq/lib/bot.py", line 591, in receive_message
message = self.__source_pipeline.receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 124, in receive
retval = self._receive()
File "/.../intelmq/intelmq/lib/pipeline.py", line 251, in _receive
raise exceptions.PipelineError(exc)
intelmq.lib.exceptions.PipelineError: pipeline failed - ValueError("Could not decode string with given encodings('utf-8',).",)
...

# Fixed code
def decode(
    text: Union[bytes, str], encodings: Sequence[str] = ("utf-8",), force: bool = False
) -> str:
    """
    Decode given string to UTF-8 (default).

    Parameters:
        text: if unicode string is given, same object is returned
        encodings: list/tuple of encodings to use
        force: Ignore invalid characters

    Returns:
        converted unicode string

    Raises:
        ValueError: if decoding failed
    """
    if isinstance(text, str):
        return text
    exception = None

    for encoding in encodings:
        try:
            return str(text.decode(encoding))
        except ValueError as exc:
            exception = exc

    if force:
        for encoding in encodings:
            try:
                return str(text.decode(encoding, "ignore"))
            except ValueError as exc:
                exception = exc

    raise DecodingError(encodings=encodings, exception=exception, object=text)
# Buggy code
def __init__(self, bot_id: str, start=False, sighup_event=None):
    super().__init__(bot_id=bot_id)
    if self.__class__.__name__ == "ParserBot":
        self.logger.error(
            "ParserBot can't be started itself. Possible Misconfiguration."
        )
        self.stop()
    self.group = "Parser"


# Traceback (TypeError)
intelmqctl run my-ip-collector
intelmqctl: Starting my-ip-collector...
Traceback (most recent call last):
File "/usr/local/bin/intelmqctl", line 11, in <module>
load_entry_point('intelmq==2.0.1a1', 'console_scripts', 'intelmqctl')()
File "/usr/local/lib/python3.6/dist-packages/intelmq/bin/intelmqctl.py", line 1425, in main
return x.run()
File "/usr/local/lib/python3.6/dist-packages/intelmq/bin/intelmqctl.py", line 912, in run
retval, results = args.func(**args_dict)
File "/usr/local/lib/python3.6/dist-packages/intelmq/bin/intelmqctl.py", line 919, in bot_run
return self.bot_process_manager.bot_run(**kwargs), None
File "/usr/local/lib/python3.6/dist-packages/intelmq/bin/intelmqctl.py", line 165, in bot_run
loglevel=loglevel)
File "/usr/local/lib/python3.6/dist-packages/intelmq/lib/bot_debugger.py", line 47, in __init__
self.instance = bot(bot_id, disable_multithreading=True)
TypeError: __init__() got an unexpected keyword argument 'disable_multithreading'

# Fixed code
def __init__(
    self, bot_id: str, start=False, sighup_event=None, disable_multithreading=None
):
    super().__init__(bot_id=bot_id)
    if self.__class__.__name__ == "ParserBot":
        self.logger.error(
            "ParserBot can't be started itself. Possible Misconfiguration."
        )
        self.stop()
    self.group = "Parser"
# Buggy code
def __init__(self, bot_id: str, start=False, sighup_event=None):
    super().__init__(bot_id=bot_id)
    if self.__class__.__name__ == "CollectorBot":
        self.logger.error(
            "CollectorBot can't be started itself. Possible Misconfiguration."
        )
        self.stop()
    self.group = "Collector"


# Traceback (TypeError)
intelmqctl run my-ip-collector
intelmqctl: Starting my-ip-collector...
Traceback (most recent call last):
File "/usr/local/bin/intelmqctl", line 11, in <module>
load_entry_point('intelmq==2.0.1a1', 'console_scripts', 'intelmqctl')()
File "/usr/local/lib/python3.6/dist-packages/intelmq/bin/intelmqctl.py", line 1425, in main
return x.run()
File "/usr/local/lib/python3.6/dist-packages/intelmq/bin/intelmqctl.py", line 912, in run
retval, results = args.func(**args_dict)
File "/usr/local/lib/python3.6/dist-packages/intelmq/bin/intelmqctl.py", line 919, in bot_run
return self.bot_process_manager.bot_run(**kwargs), None
File "/usr/local/lib/python3.6/dist-packages/intelmq/bin/intelmqctl.py", line 165, in bot_run
loglevel=loglevel)
File "/usr/local/lib/python3.6/dist-packages/intelmq/lib/bot_debugger.py", line 47, in __init__
self.instance = bot(bot_id, disable_multithreading=True)
TypeError: __init__() got an unexpected keyword argument 'disable_multithreading'

# Fixed code
def __init__(
    self, bot_id: str, start=False, sighup_event=None, disable_multithreading=None
):
    super().__init__(bot_id=bot_id)
    if self.__class__.__name__ == "CollectorBot":
        self.logger.error(
            "CollectorBot can't be started itself. Possible Misconfiguration."
        )
        self.stop()
    self.group = "Collector"
# Buggy code
def get_domain(self, address):
    try:
        dom = re.search(
            r"(//|^)([a-z0-9.-]*[a-z]\.[a-z][a-z-]*?(?:[/:].*|$))", address
        ).group(2)
        if not self.in_whitelist(dom):
            if get_tld(url_normalize(dom), fail_silently=True):
                return url_normalize(dom)
        return None
    except AttributeError:
        return None
    except (
        UnicodeError
    ):  # url_normalize's error, happens when something weird matches regex
        self.logger.info("Caught UnicodeError on %r.", address)
        return None


# Traceback (AssertionError)
INTELMQ_TEST_EXOTIC=1 nosetests tests/bots/parsers/twitter/test_parser.py
.F
======================================================================
FAIL: test_parse (intelmq.tests.bots.parsers.twitter.test_parser.TestTwitterParserBot)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/sebastian/dev/intelmq/intelmq/tests/bots/parsers/twitter/test_parser.py", line 71, in test_parse
self.assertMessageEqual(0, EVENTS[0])
File "/home/sebastian/dev/intelmq/intelmq/lib/test.py", line 481, in assertMessageEqual
self.assertDictEqual(expected, event_dict)
AssertionError: {'source.url': 'http://testweb.com/sales-in[486 chars]20K'} != {'feed.name': 'Twitter', 'feed.url': 'https[487 chars]ent'}
{'__type': 'Event',
'classification.type': 'blacklist',
'feed.name': 'Twitter',
'feed.url': 'https://twitter.com/El_carlos/1234456',
'raw': 'SGVyZSBpcyBteSBsaW5rLiBJIGhhdmUgcnVuIFNhbGVzLmV4ZSB3aXRoIFZNLgoKaHh4cDovL3Rlc3R3ZWJbLl1jb20vU2FsZXMtSW52b2ljZS8KY2MucHJvL2ltYWdlcy5odG1sClRoZXJlIGlzIGFub3RoZXIgdGhyZWF0LCBhbmQgdGhhdCBpcyBhdCAKaHR0cF1dOi8va2FybG9zIC5uZXQvCgp0YWsgYSBsb29rIGF0IGh4eHA6Ly9ibG9ja1suXWRlWy5dY29tL21hbHdhcmUuZXhlCkFub3RoZXIgdGhyZWF0IGlzIHBsYWNlZCBhdCAgZ2h6eixjb20K',
-  'source.url': 'http://testweb.com/sales-invoice/'}
+  'source.url': 'https://testweb.com/sales-invoice/'}
?                     +

-------------------- >> begin captured logging << --------------------
test-bot: INFO: TwitterParserBot initialized with id test-bot and intelmq 1.1.1.alpha1 and python 3.6.5 (default, Mar 31 2018, 19:45:04) [GCC] as process 3356.
test-bot: INFO: Bot is starting.
test-bot: INFO: Processed 5 messages since last logging.
test-bot: INFO: Bot stopped.
--------------------- >> end captured logging << ---------------------

----------------------------------------------------------------------
Ran 2 tests in 0.041s

FAILED (failures=1)

# Fixed code
def get_domain(self, address):
    try:
        dom = re.search(
            r"(//|^)([a-z0-9.-]*[a-z]\.[a-z][a-z-]*?(?:[/:].*|$))", address
        ).group(2)
        if not self.in_whitelist(dom):
            if get_tld(url_normalize(dom, **self.url_kwargs), fail_silently=True):
                return url_normalize(dom, **self.url_kwargs)
        return None
    except AttributeError:
        return None
    except (
        UnicodeError
    ):  # url_normalize's error, happens when something weird matches regex
        self.logger.info("Caught UnicodeError on %r.", address)
        return None
# Buggy code
def get_linecol(model_obj, as_dict=False):
    """Gets the position of a model object in the sieve file.

    Args:
        model_obj: the model object
        as_dict: return the position as a dict instead of a tuple.

    Returns:
        Returns the line and column number for the model object's position in the sieve file.
        Default return type is a tuple of (line,col). Optionally, returns a dict when as_dict == True.

    """
    metamodel = textx.model.metamodel(model_obj)
    tup = metamodel.parser.pos_to_linecol(model_obj._tx_position)
    if as_dict:
        return dict(zip(["line", "col"], tup))
    return tup


# Traceback (AttributeError)
Traceback (most recent call last):
File "/home/travis/build/certtools/intelmq/intelmq/lib/bot.py", line 163, in start
self.process()
File "/home/travis/build/certtools/intelmq/intelmq/bots/experts/sieve/expert.py", line 86, in process
procedure = self.process_rule(rule, event)
File "/home/travis/build/certtools/intelmq/intelmq/bots/experts/sieve/expert.py", line 104, in process_rule
self.logger.debug('Matched event based on rule at %s: %s.', self.get_linecol(rule.if_), event)
File "/home/travis/build/certtools/intelmq/intelmq/bots/experts/sieve/expert.py", line 323, in get_linecol
metamodel = textx.model.metamodel(model_obj)
AttributeError: module 'textx.model' has no attribute 'metamodel'

# Fixed code
def get_linecol(model_obj, as_dict=False):
    """Gets the position of a model object in the sieve file.

    Args:
        model_obj: the model object
        as_dict: return the position as a dict instead of a tuple.

    Returns:
        Returns the line and column number for the model object's position in the sieve file.
        Default return type is a tuple of (line,col). Optionally, returns a dict when as_dict == True.

    """
    # The __version__ attribute is first available with version 1.7.0
    if hasattr(textx, "__version__"):
        parser = textx.model.get_model(model_obj)._tx_parser
    else:
        parser = textx.model.metamodel(model_obj).parser
    tup = parser.pos_to_linecol(model_obj._tx_position)
    if as_dict:
        return dict(zip(["line", "col"], tup))
    return tup
# Buggy code
def stop(self, exitcode: int = 1):
    try:
        self.shutdown()
    except BaseException:
        pass

    if self.__message_counter:
        self.logger.info(
            "%s %d messages since last logging.",
            self._message_processed_verb,
            self.__message_counter,
        )

    self.__disconnect_pipelines()

    if self.logger:
        self.logger.info("Bot stopped.")
        logging.shutdown()
    else:
        self.__log_buffer.append(("info", "Bot stopped."))
        self.__print_log_buffer()

    if not getattr(self.parameters, "testing", False):
        del self
        exit(exitcode)


# Traceback (AttributeError)
2017-05-08 05:38:52,150 - xmpp-bsi-collector - INFO - Idling for 3600.0s now.
2017-05-08 06:25:17,283 - xmpp-bsi-collector - INFO - Received SIGHUP, initializing again later.
2017-05-08 06:25:17,283 - xmpp-bsi-collector - INFO - Handling SIGHUP, initializing again now.
2017-05-08 06:25:17,284 - xmpp-bsi-collector - INFO - XMPPCollectorBot initialized with id xmpp-bsi-collector and version 3.4.2 (default, Oct  8 2014, 10:45:20) as process 7050.
2017-05-08 06:25:17,284 - xmpp-bsi-collector - INFO - Bot is starting.
2017-05-08 06:25:17,286 - xmpp-bsi-collector - INFO - Initiated.
2017-05-08 06:25:17,294 - xmpp-bsi-collector - ERROR - Could not connect to XMPP-Server.
2017-05-08 06:25:17,294 - xmpp-bsi-collector - ERROR - Could not disconnect from XMPP Server.
2017-05-08 06:25:17,294 - xmpp-bsi-collector - INFO - Bot stopped.
2017-05-10 08:03:32,964 - xmpp-bsi-collector - ERROR - Error during message handling.
Traceback (most recent call last):
File "/home/aaron/intelmq/intelmq/bots/collectors/xmpp/collector.py", line 198, in log_message
self.send_message(report)
File "/home/aaron/intelmq/intelmq/lib/bot.py", line 677, in send_message
super(CollectorBot, self).send_message(*messages)
File "/home/aaron/intelmq/intelmq/lib/bot.py", line 343, in send_message
self.__destination_pipeline.send(raw_message)
AttributeError: 'NoneType' object has no attribute 'send'

# Fixed code
def stop(self, exitcode: int = 1):
    try:
        self.shutdown()
    except BaseException:
        pass

    if self.__message_counter:
        self.logger.info(
            "%s %d messages since last logging.",
            self._message_processed_verb,
            self.__message_counter,
        )

    self.__disconnect_pipelines()

    if self.logger:
        self.logger.info("Bot stopped.")
        logging.shutdown()
        # Bots using threads that do not exit properly, see #970
        if self.__class__.__name__ in ["XMPPCollectorBot", "XMPPOutputBot"]:
            proc = psutil.Process(os.getpid())
            proc.send_signal(signal.SIGTERM)
    else:
        self.__log_buffer.append(("info", "Bot stopped."))
        self.__print_log_buffer()

    if not getattr(self.parameters, "testing", False):
        del self
        exit(exitcode)
# Buggy code
def sanitize(value):
    value = value.strip(".")
    if value:
        return value.encode("idna").decode().lower()


# Traceback (UnicodeError)
Traceback (most recent call last):
File ".../intelmq/intelmq/lib/bot.py", line 153, in start
self.process()
File ".../intelmq/intelmq/bots/parsers/spamhaus/parser_cert.py",
line 59, in process
event.add('destination.fqdn', row_splitted[5], raise_failure=False)
File ".../intelmq/intelmq/lib/message.py", line 194, in add
value = self.__sanitize_value(key, value)
File ".../intelmq/intelmq/lib/message.py", line 281, in __sanitize_value
return class_reference().sanitize(value)
File ".../intelmq/intelmq/lib/harmonization.py", line 381, in sanitize
return value.encode('idna').decode().lower()
UnicodeError: encoding with 'idna' codec failed (UnicodeError: label empty or too long)

# Fixed code
def sanitize(value):
    value = value.strip(".")
    if value:
        try:
            return value.encode("idna").decode().lower()
        except UnicodeError:
            return
# Buggy code
def parse(self, report):
    raw_report = utils.base64_decode(report.get("raw"))
    # ignore lines starting with #
    raw_report = re.sub(r"(?m)^#.*\n?", "", raw_report)
    # ignore null bytes
    raw_report = re.sub(r"(?m)\0", "", raw_report)
    # skip header
    if getattr(self.parameters, "skip_header", False):
        raw_report = raw_report[raw_report.find("\n") + 1 :]
    for row in csv.reader(
        io.StringIO(raw_report), delimiter=str(self.parameters.delimiter)
    ):
        yield row


# Traceback (_csv.Error)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib64/python3.4/csv.py", line 110, in __next__
row = next(self.reader)
_csv.Error: line contains NULL byte

# Fixed code
def parse(self, report):
    raw_report = utils.base64_decode(report.get("raw"))
    raw_report = raw_report.translate({0: None})
    # ignore lines starting with #
    raw_report = re.sub(r"(?m)^#.*\n?", "", raw_report)
    # ignore null bytes
    raw_report = re.sub(r"(?m)\0", "", raw_report)
    # skip header
    if getattr(self.parameters, "skip_header", False):
        raw_report = raw_report[raw_report.find("\n") + 1 :]
    for row in csv.reader(
        io.StringIO(raw_report), delimiter=str(self.parameters.delimiter)
    ):
        yield row
# Buggy code
def process(self):
    report = self.receive_message()

    columns = [
        "__IGNORE__",
        "source.url",
        "event_description.url",
        "time.source",
        "__IGNORE__",
        "__IGNORE__",
        "__IGNORE__",
        "event_description.target",
    ]

    raw_report = utils.base64_decode(report.get("raw"))
    for row in csv.reader(io.StringIO(raw_report)):
        if not len(row):  # csv module can give empty lists
            self.acknowledge_message()
            return

        # ignore headers
        if "phish_id" in row:
            continue

        event = self.new_event(report)
        event.change("feed.url", event["feed.url"][: event["feed.url"].find("data/")])

        for key, value in zip(columns, row):
            if key == "__IGNORE__":
                continue

            event.add(key, value)

        event.add("classification.type", "phishing")
        event.add("raw", ",".join(row))

        self.send_message(event)
    self.acknowledge_message()


# Traceback (_csv.Error)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib64/python3.4/csv.py", line 110, in __next__
row = next(self.reader)
_csv.Error: line contains NULL byte

# Fixed code
def process(self):
    report = self.receive_message()

    columns = [
        "__IGNORE__",
        "source.url",
        "event_description.url",
        "time.source",
        "__IGNORE__",
        "__IGNORE__",
        "__IGNORE__",
        "event_description.target",
    ]

    raw_report = utils.base64_decode(report.get("raw"))
    raw_report = raw_report.translate({0: None})
    for row in csv.reader(io.StringIO(raw_report)):
        if not len(row):  # csv module can give empty lists
            self.acknowledge_message()
            return

        # ignore headers
        if "phish_id" in row:
            continue

        event = self.new_event(report)
        event.change("feed.url", event["feed.url"][: event["feed.url"].find("data/")])

        for key, value in zip(columns, row):
            if key == "__IGNORE__":
                continue

            event.add(key, value)

        event.add("classification.type", "phishing")
        event.add("raw", ",".join(row))

        self.send_message(event)
    self.acknowledge_message()
# Buggy code
def parse(self, report):
    raw_report = utils.base64_decode(report["raw"])
    # Temporary fix for https://github.com/certtools/intelmq/issues/967
    raw_report = raw_report.translate({0: None})
    csvr = csv.DictReader(io.StringIO(raw_report))

    # create an array of fieldnames,
    # those were automagically created by the dictreader
    self.fieldnames = csvr.fieldnames

    for row in csvr:
        yield row


# Traceback (_csv.Error)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib64/python3.4/csv.py", line 110, in __next__
row = next(self.reader)
_csv.Error: line contains NULL byte

# Fixed code
def parse(self, report):
    raw_report = utils.base64_decode(report["raw"])
    raw_report = raw_report.translate({0: None})
    csvr = csv.DictReader(io.StringIO(raw_report))

    # create an array of fieldnames,
    # those were automagically created by the dictreader
    self.fieldnames = csvr.fieldnames

    for row in csvr:
        yield row