gurudesh/pytracebugs huggingface dataset:
# Buggy code
def __init__(self, msg=str()):
    self.msg = msg
    super(WebHdfsException, self).__init__(self.msg)


# Traceback (TypeError)
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-40-b63bd711d13c> in <module>
----> 1 smart_copy('./test_file.txt', 'webhdfs://XXXX@XXX.XXX.XXX.XXX:XXXXX/user/XXXX/smart_copy/test_file.txt')

<ipython-input-38-694f70cf0776> in smart_copy(source_file, sync_file)
3     '''
4     with open(source_file, 'rb') as source:
----> 5         with open(sync_file, 'wb') as sync:
6             for line in source:
7                 sync.write(line)

~/Github/tools/.venv/lib/python3.7/site-packages/smart_open/smart_open_lib.py in open(uri, mode, buffering, encoding, errors, newline, closefd, opener, ignore_ext, transport_params)
346     except KeyError:
347         binary_mode = mode
--> 348     binary, filename = _open_binary_stream(uri, binary_mode, transport_params)
349     if ignore_ext:
350         decompressed = binary

~/Github/tools/.venv/lib/python3.7/site-packages/smart_open/smart_open_lib.py in _open_binary_stream(uri, mode, transport_params)
560         elif parsed_uri.scheme == "webhdfs":
561             kw = _check_kwargs(smart_open_webhdfs.open, transport_params)
--> 562             return smart_open_webhdfs.open(parsed_uri.uri_path, mode, **kw), filename
563         elif parsed_uri.scheme.startswith('http'):
564             #

~/Github/tools/.venv/lib/python3.7/site-packages/smart_open/webhdfs.py in open(uri, mode, min_part_size)
40         return BufferedInputBase(uri)
41     elif mode == 'wb':
---> 42         return BufferedOutputBase(uri, min_part_size=min_part_size)
43     else:
44         raise NotImplementedError('webhdfs support for mode %r not implemented' % mode)

~/Github/tools/.venv/lib/python3.7/site-packages/smart_open/webhdfs.py in __init__(self, uri_path, min_part_size)
129                                      params=payload, allow_redirects=False)
130         if not init_response.status_code == httplib.TEMPORARY_REDIRECT:
--> 131             raise WebHdfsException(str(init_response.status_code) + "\n" + init_response.content)
132         uri = init_response.headers['location']
133         response = requests.put(uri, data="", headers={'content-type': 'application/octet-stream'})

TypeError: can only concatenate str (not "bytes") to str

# Fixed code
def __init__(self, msg="", status_code=None):
    self.msg = msg
    self.status_code = status_code
    super(WebHdfsException, self).__init__(repr(self))
# Buggy code
def _parse_uri_file(parsed_uri):
    assert parsed_uri.scheme in (None, "", "file")
    uri_path = parsed_uri.netloc + parsed_uri.path
    # '~/tmp' may be expanded to '/Users/username/tmp'
    uri_path = os.path.expanduser(uri_path)

    if not uri_path:
        raise RuntimeError("invalid file URI: %s" % str(parsed_uri))

    return Uri(scheme="file", uri_path=uri_path)


# Traceback (FileNotFoundError)
smart_open.smart_open('//anaconda/envs/python3/lib/python3.5/site-packages/gensim/test/test_data/lee_background.cor','r')
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-12-6a05871c0775> in <module>()
----> 1 smart_open.smart_open('//anaconda/envs/python3/lib/python3.5/site-packages/gensim/test/test_data/lee_background.cor','r')

//anaconda/envs/python3/lib/python3.5/site-packages/smart_open/smart_open_lib.py in smart_open(uri, mode, **kw)
138             # local files -- both read &amp; write supported
139             # compression, if any, is determined by the filename extension (.gz, .bz2)
--> 140             return file_smart_open(parsed_uri.uri_path, mode)
141         elif parsed_uri.scheme in ("s3", "s3n", "s3u"):
142             kwargs = {}

//anaconda/envs/python3/lib/python3.5/site-packages/smart_open/smart_open_lib.py in file_smart_open(fname, mode)
642
643     """
--> 644     return compression_wrapper(open(fname, mode), fname, mode)
645
646

FileNotFoundError: [Errno 2] No such file or directory: 'anaconda/envs/python3/lib/python3.5/site-packages/gensim/test/test_data/lee_background.cor'

# Fixed code
def _parse_uri_file(input_path):
    # '~/tmp' may be expanded to '/Users/username/tmp'
    uri_path = os.path.expanduser(input_path)

    if not uri_path:
        raise RuntimeError("invalid file URI: %s" % input_path)

    return Uri(scheme="file", uri_path=uri_path)
# Buggy code
def open(bucket_id, key_id, mode, **kwargs):
    logger.debug("%r", locals())
    if mode not in MODES:
        raise NotImplementedError("bad mode: %r expected one of %r" % (mode, MODES))

    buffer_size = kwargs.pop("buffer_size", io.DEFAULT_BUFFER_SIZE)
    encoding = kwargs.pop("encoding", "utf-8")
    errors = kwargs.pop("errors", None)
    newline = kwargs.pop("newline", None)
    line_buffering = kwargs.pop("line_buffering", False)
    s3_min_part_size = kwargs.pop("s3_min_part_size", DEFAULT_MIN_PART_SIZE)

    if mode in (READ, READ_BINARY):
        fileobj = BufferedInputBase(bucket_id, key_id, **kwargs)
    elif mode in (WRITE, WRITE_BINARY):
        fileobj = BufferedOutputBase(
            bucket_id, key_id, min_part_size=s3_min_part_size, **kwargs
        )
    else:
        assert False

    if mode in (READ, WRITE):
        return io.TextIOWrapper(
            fileobj,
            encoding=encoding,
            errors=errors,
            newline=newline,
            line_buffering=line_buffering,
        )
    elif mode in (READ_BINARY, WRITE_BINARY):
        return fileobj
    else:
        assert False


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def open(bucket_id, key_id, mode, **kwargs):
    logger.debug("%r", locals())
    if mode not in MODES:
        raise NotImplementedError("bad mode: %r expected one of %r" % (mode, MODES))

    encoding = kwargs.pop("encoding", "utf-8")
    errors = kwargs.pop("errors", None)
    newline = kwargs.pop("newline", None)
    line_buffering = kwargs.pop("line_buffering", False)
    s3_min_part_size = kwargs.pop("s3_min_part_size", DEFAULT_MIN_PART_SIZE)

    if mode in (READ, READ_BINARY):
        fileobj = SeekableBufferedInputBase(bucket_id, key_id, **kwargs)
    elif mode in (WRITE, WRITE_BINARY):
        fileobj = BufferedOutputBase(
            bucket_id, key_id, min_part_size=s3_min_part_size, **kwargs
        )
    else:
        assert False

    if mode in (READ, WRITE):
        return io.TextIOWrapper(
            fileobj,
            encoding=encoding,
            errors=errors,
            newline=newline,
            line_buffering=line_buffering,
        )
    elif mode in (READ_BINARY, WRITE_BINARY):
        return fileobj
    else:
        assert False
# Buggy code
def __init__(self, s3_object):
    self.position = 0
    self._object = s3_object
    self._content_length = self._object.content_length


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def __init__(self, s3_object):
    self.position = 0
    self._object = s3_object
    self._body = s3_object.get()["Body"]
# Buggy code
def read(self, size=-1):
    if self.position == self._content_length:
        return b""
    if size <= 0:
        end = None
    else:
        end = min(self._content_length, self.position + size)
    range_string = _range_string(self.position, stop=end)
    logger.debug("range_string: %r", range_string)
    body = self._object.get(Range=range_string)["Body"].read()
    self.position += len(body)
    return body


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def read(self, size=-1):
    if size == -1:
        return self._body.read()
    return self._body.read(size)
# Buggy code
def __init__(self, bucket, key, **kwargs):
    session = boto3.Session(profile_name=kwargs.pop("profile_name", None))
    s3 = session.resource("s3", **kwargs)
    self._object = s3.Object(bucket, key)
    self._raw_reader = RawReader(self._object)
    self._content_length = self._object.content_length
    self._current_pos = 0
    self._buffer = b""
    self._eof = False

    #
    # This member is part of the io.BufferedIOBase interface.
    #
    self.raw = None


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def __init__(
    self,
    bucket,
    key,
    buffer_size=DEFAULT_BUFFER_SIZE,
    line_terminator=BINARY_NEWLINE,
    **kwargs,
):
    session = boto3.Session(profile_name=kwargs.pop("profile_name", None))
    s3 = session.resource("s3", **kwargs)
    self._object = s3.Object(bucket, key)
    self._raw_reader = RawReader(self._object)
    self._content_length = self._object.content_length
    self._current_pos = 0
    self._buffer = b""
    self._eof = False
    self._buffer_size = buffer_size
    self._line_terminator = line_terminator

    #
    # This member is part of the io.BufferedIOBase interface.
    #
    self.raw = None
# Buggy code
def seekable(self):
    """If False, seek(), tell() and truncate() will raise IOError.

    We offer only seek support, and no truncate support."""
    return True


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def seekable(self):
    return False
# Buggy code
def read(self, size=-1):
    """Read up to size bytes from the object and return them."""
    if size <= 0:
        if len(self._buffer):
            from_buf = self._read_from_buffer(len(self._buffer))
        else:
            from_buf = b""
        self._current_pos = self._content_length
        return from_buf + self._raw_reader.read()

    #
    # Return unused data first
    #
    if len(self._buffer) >= size:
        return self._read_from_buffer(size)

    #
    # If the stream is finished, return what we have.
    #
    if self._eof:
        return self._read_from_buffer(len(self._buffer))

    #
    # Fill our buffer to the required size.
    #
    # logger.debug('filling %r byte-long buffer up to %r bytes', len(self._buffer), size)
    while len(self._buffer) < size and not self._eof:
        raw = self._raw_reader.read(size=io.DEFAULT_BUFFER_SIZE)
        if len(raw):
            self._buffer += raw
        else:
            logger.debug("reached EOF while filling buffer")
            self._eof = True

    return self._read_from_buffer(size)


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def read(self, size=-1):
    """Read up to size bytes from the object and return them."""
    if size <= 0:
        if len(self._buffer):
            from_buf = self._read_from_buffer(len(self._buffer))
        else:
            from_buf = b""
        self._current_pos = self._content_length
        return from_buf + self._raw_reader.read()

    #
    # Return unused data first
    #
    if len(self._buffer) >= size:
        return self._read_from_buffer(size)

    #
    # If the stream is finished, return what we have.
    #
    if self._eof:
        return self._read_from_buffer(len(self._buffer))

    #
    # Fill our buffer to the required size.
    #
    # logger.debug('filling %r byte-long buffer up to %r bytes', len(self._buffer), size)
    self._fill_buffer(size)
    return self._read_from_buffer(size)
# Buggy code
def __init__(self, bucket, key, min_part_size=DEFAULT_MIN_PART_SIZE, **kwargs):
    if min_part_size < MIN_MIN_PART_SIZE:
        logger.warning(
            "S3 requires minimum part size >= 5MB; \
multipart upload may fail"
        )

    session = boto3.Session(profile_name=kwargs.pop("profile_name", None))
    s3 = session.resource("s3", **kwargs)

    #
    # https://stackoverflow.com/questions/26871884/how-can-i-easily-determine-if-a-boto-3-s3-bucket-resource-exists
    #
    s3.create_bucket(Bucket=bucket)
    self._object = s3.Object(bucket, key)
    self._min_part_size = min_part_size
    self._mp = self._object.initiate_multipart_upload()

    self._buf = io.BytesIO()
    self._total_bytes = 0
    self._total_parts = 0
    self._parts = []

    #
    # This member is part of the io.BufferedIOBase interface.
    #
    self.raw = None


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def __init__(self, bucket, key, min_part_size=DEFAULT_MIN_PART_SIZE, **kwargs):
    if min_part_size < MIN_MIN_PART_SIZE:
        logger.warning(
            "S3 requires minimum part size >= 5MB; \
multipart upload may fail"
        )

    session = boto3.Session(profile_name=kwargs.pop("profile_name", None))
    s3 = session.resource("s3", **kwargs)

    #
    # https://stackoverflow.com/questions/26871884/how-can-i-easily-determine-if-a-boto-3-s3-bucket-resource-exists
    #
    try:
        s3.meta.client.head_bucket(Bucket=bucket)
    except botocore.client.ClientError:
        raise ValueError(
            "the bucket %r does not exist, or is forbidden for access" % bucket
        )
    self._object = s3.Object(bucket, key)
    self._min_part_size = min_part_size
    self._mp = self._object.initiate_multipart_upload()

    self._buf = io.BytesIO()
    self._total_bytes = 0
    self._total_parts = 0
    self._parts = []

    #
    # This member is part of the io.BufferedIOBase interface.
    #
    self.raw = None
# Buggy code
def s3_open_key(key, mode, **kwargs):
    logger.debug("%r", locals())
    #
    # TODO: handle boto3 keys as well
    #
    host = kwargs.pop("host", None)
    if host is not None:
        kwargs["endpoint_url"] = "http://" + host

    if kwargs.pop("ignore_extension", False):
        codec = None
    else:
        codec = _detect_codec(key.name)

    #
    # Codecs work on a byte-level, so the underlying S3 object should
    # always be reading bytes.
    #
    if codec and mode in (smart_open_s3.READ, smart_open_s3.READ_BINARY):
        s3_mode = smart_open_s3.READ_BINARY
    elif codec and mode in (smart_open_s3.WRITE, smart_open_s3.WRITE_BINARY):
        s3_mode = smart_open_s3.WRITE_BINARY
    else:
        s3_mode = mode

    logging.debug("codec: %r mode: %r s3_mode: %r", codec, mode, s3_mode)
    fobj = smart_open_s3.open(key.bucket.name, key.name, s3_mode, **kwargs)
    return _CODECS[codec](fobj, mode)


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def s3_open_key(key, mode, **kwargs):
    logger.debug("%r", locals())
    #
    # TODO: handle boto3 keys as well
    #
    host = kwargs.pop("host", None)
    if host is not None:
        kwargs["endpoint_url"] = "http://" + host

    if kwargs.pop("ignore_extension", False):
        codec = None
    else:
        codec = _detect_codec(key.name)

    #
    # Codecs work on a byte-level, so the underlying S3 object should
    # always be reading bytes.
    #
    if mode in (smart_open_s3.READ, smart_open_s3.READ_BINARY):
        s3_mode = smart_open_s3.READ_BINARY
    elif mode in (smart_open_s3.WRITE, smart_open_s3.WRITE_BINARY):
        s3_mode = smart_open_s3.WRITE_BINARY
    else:
        raise NotImplementedError("mode %r not implemented for S3" % mode)

    logging.debug("codec: %r mode: %r s3_mode: %r", codec, mode, s3_mode)
    encoding = kwargs.get("encoding")
    errors = kwargs.get("errors", DEFAULT_ERRORS)
    fobj = smart_open_s3.open(key.bucket.name, key.name, s3_mode, **kwargs)
    decompressed_fobj = _CODECS[codec](fobj, mode)
    decoded_fobj = encoding_wrapper(
        decompressed_fobj, mode, encoding=encoding, errors=errors
    )
    return decoded_fobj
# Buggy code
def _wrap_gzip(fileobj, mode):
    return contextlib.closing(gzip.GzipFile(fileobj=fileobj, mode=mode))


# Traceback (TypeError)
(smartopen)sergeyich:issue152 misha$ git rev-parse --short HEAD
d10166c
(smartopen)sergeyich:issue152 misha$ time python reproduce.py s3://commoncrawl/crawl-002/2010/09/25/0/1285411480200_0.arc.gz
0it [00:00, ?it/s]Traceback (most recent call last):
File "reproduce.py", line 5, in <module>
for i, _ in enumerate(tqdm(smart_open(sys.argv[1], 'rb'))):
File "/Users/misha/envs/smartopen/lib/python3.6/site-packages/tqdm/_tqdm.py", line 953, in __iter__
for obj in iterable:
TypeError: 'closing' object is not iterable


real    0m3.131s
user    0m0.906s
sys     0m0.151s

# Fixed code
def _wrap_gzip(fileobj, mode):
    return gzip.GzipFile(fileobj=fileobj, mode=mode)
# Buggy code
def _emit_duration_stats_for_finished_state(self):
    if self.state == State.RUNNING:
        return

    duration = self.end_date - self.start_date
    if self.state is State.SUCCESS:
        Stats.timing(f"dagrun.duration.success.{self.dag_id}", duration)
    elif self.state == State.FAILED:
        Stats.timing(f"dagrun.duration.failed.{self.dag_id}", duration)


# Traceback (TypeError)
Exception when executing SchedulerJob._run_scheduler_loop

Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1280, in _execute
self._run_scheduler_loop()
File "/usr/local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1382, in _run_scheduler_loop
num_queued_tis = self._do_scheduling(session)
File "/usr/local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1521, in _do_scheduling
self._schedule_dag_run(dag_run, active_runs_by_dag_id.get(dag_run.dag_id, set()), session)
File "/usr/local/lib/python3.8/site-packages/airflow/jobs/scheduler_job.py", line 1760, in _schedule_dag_run
schedulable_tis, callback_to_run = dag_run.update_state(session=session, execute_callbacks=False)
File "/usr/local/lib/python3.8/site-packages/airflow/utils/session.py", line 62, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/airflow/models/dagrun.py", line 478, in update_state
self._emit_duration_stats_for_finished_state()
File "/usr/local/lib/python3.8/site-packages/airflow/models/dagrun.py", line 615, in _emit_duration_stats_for_finished_state
duration = self.end_date - self.start_date
TypeError: unsupported operand type(s) for -: 'datetime.datetime' and 'NoneType'

# Fixed code
def _emit_duration_stats_for_finished_state(self):
    if self.state == State.RUNNING:
        return
    if self.start_date is None:
        self.log.warning(
            "Failed to record duration of %s: start_date is not set.", self
        )
        return
    if self.end_date is None:
        self.log.warning("Failed to record duration of %s: end_date is not set.", self)
        return

    duration = self.end_date - self.start_date
    if self.state is State.SUCCESS:
        Stats.timing(f"dagrun.duration.success.{self.dag_id}", duration)
    elif self.state == State.FAILED:
        Stats.timing(f"dagrun.duration.failed.{self.dag_id}", duration)
# Buggy code
def clear(args):
    logging.basicConfig(level=settings.LOGGING_LEVEL, format=settings.SIMPLE_LOG_FORMAT)
    dags = get_dags(args)

    if args.task_regex:
        for idx, dag in enumerate(dags):
            dags[idx] = dag.sub_dag(
                task_regex=args.task_regex,
                include_downstream=args.downstream,
                include_upstream=args.upstream,
            )

    DAG.clear_dags(
        dags,
        start_date=args.start_date,
        end_date=args.end_date,
        only_failed=args.only_failed,
        only_running=args.only_running,
        confirm_prompt=not args.no_confirm,
        include_subdags=not args.exclude_subdags,
        include_parentdag=not args.exclude_parentdag,
    )


# Traceback (AttributeError)
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 37, in <module>
args.func(args)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/cli.py", line 233, in wrapper
func(args)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/cli.py", line 81, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py", line 867, in clear
confirm_prompt=not args.no_confirm,
AttributeError: 'Namespace' object has no attribute 'no_confirm'

# Fixed code
def clear(args):
    logging.basicConfig(level=settings.LOGGING_LEVEL, format=settings.SIMPLE_LOG_FORMAT)
    dags = get_dags(args)

    if args.task_regex:
        for idx, dag in enumerate(dags):
            dags[idx] = dag.sub_dag(
                task_regex=args.task_regex,
                include_downstream=args.downstream,
                include_upstream=args.upstream,
            )
    if args.yes:
        args.no_confirm = args.yes
    DAG.clear_dags(
        dags,
        start_date=args.start_date,
        end_date=args.end_date,
        only_failed=args.only_failed,
        only_running=args.only_running,
        confirm_prompt=not args.no_confirm,
        include_subdags=not args.exclude_subdags,
        include_parentdag=not args.exclude_parentdag,
    )
# Buggy code
def _check_file(self, file_path):
    problems = []
    class_name_to_check = self.MACRO_PLUGIN_CLASS.split(".")[-1]
    with open(file_path, "r") as file_pointer:
        for line_number, line in enumerate(file_pointer, 1):
            if class_name_to_check in line:
                problems.append(self._change_info(file_path, line_number))
    return problems


# Traceback (UnicodeDecodeError)
========================================================================================================================================================== STATUS ==========================================================================================================================================================

Check for latest versions of apache-airflow and checker...........................................................................................................................................................................................................................................................SUCCESS
Traceback (most recent call last):
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/bin/airflow", line 37, in <module>
args.func(args)
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/checker.py", line 88, in run
all_problems = check_upgrade(formatter, rules)
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/checker.py", line 37, in check_upgrade
rule_status = RuleStatus.from_rule(rule)
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/problem.py", line 44, in from_rule
result = rule.check()
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/rules/airflow_macro_plugin_removed.py", line 52, in check
problems.extend(self._check_file(file_path))
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/rules/airflow_macro_plugin_removed.py", line 42, in _check_file
for line_number, line in enumerate(file_pointer, 1):
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/codecs.py", line 322, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x82 in position 16: invalid start byte

# Fixed code
def _check_file(self, file_path):
    problems = []
    class_name_to_check = self.MACRO_PLUGIN_CLASS.split(".")[-1]
    with open(file_path, "r") as file_pointer:
        try:
            for line_number, line in enumerate(file_pointer, 1):
                if class_name_to_check in line:
                    problems.append(self._change_info(file_path, line_number))
        except UnicodeDecodeError:
            problems.append("Unable to read python file {}".format(file_path))
    return problems
# Buggy code
def check(self):
    dag_folder = conf.get("core", "dags_folder")
    file_paths = list_py_file_paths(directory=dag_folder, include_examples=False)
    problems = []
    for file_path in file_paths:
        problems.extend(self._check_file(file_path))
    return problems


# Traceback (UnicodeDecodeError)
========================================================================================================================================================== STATUS ==========================================================================================================================================================

Check for latest versions of apache-airflow and checker...........................................................................................................................................................................................................................................................SUCCESS
Traceback (most recent call last):
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/bin/airflow", line 37, in <module>
args.func(args)
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/checker.py", line 88, in run
all_problems = check_upgrade(formatter, rules)
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/checker.py", line 37, in check_upgrade
rule_status = RuleStatus.from_rule(rule)
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/problem.py", line 44, in from_rule
result = rule.check()
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/rules/airflow_macro_plugin_removed.py", line 52, in check
problems.extend(self._check_file(file_path))
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/rules/airflow_macro_plugin_removed.py", line 42, in _check_file
for line_number, line in enumerate(file_pointer, 1):
File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/codecs.py", line 322, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x82 in position 16: invalid start byte

# Fixed code
def check(self):
    dag_folder = conf.get("core", "dags_folder")
    file_paths = list_py_file_paths(directory=dag_folder, include_examples=False)
    problems = []
    for file_path in file_paths:
        if not file_path.endswith(".py"):
            continue
        problems.extend(self._check_file(file_path))
    return problems
# Buggy code
def _add_callback_to_queue(self, request: CallbackRequest):
    self._callback_to_execute[request.full_filepath].append(request)
    # Callback has a higher priority over DAG Run scheduling
    if request.full_filepath in self._file_path_queue:
        self._file_path_queue.remove(request.full_filepath)
    self._file_path_queue.insert(0, request.full_filepath)


# Traceback (KeyError)
[2020-12-13 19:35:33,752] {dagbag.py:440} INFO - Filling up the DagBag from /usr/local/lib/python3.8/site-packages/airflow/example_dags/example_bash_operator.py
Running <TaskInstance: example_bash_operator.run_after_loop 2020-12-13T19:35:30.648020+00:00 [queued]> on host 6611da4b1a27
[2020-12-13 19:35:34,517] {dagrun.py:444} INFO - Marking run <DagRun example_bash_operator @ 2020-12-13 19:35:30.648020+00:00: manual__2020-12-13T19:35:30.648020+00:00, externally triggered: True> successful
[2020-12-13 19:35:34,523] {scheduler_job.py:1193} INFO - Executor reports execution of example_bash_operator.run_after_loop execution_date=2020-12-13 19:35:30.648020+00:00 exited with status success for try_number 1
Process ForkProcess-34:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/local/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.8/site-packages/airflow/utils/dag_processing.py", line 365, in _run_processor_manager
processor_manager.start()
File "/usr/local/lib/python3.8/site-packages/airflow/utils/dag_processing.py", line 596, in start
return self._run_parsing_loop()
File "/usr/local/lib/python3.8/site-packages/airflow/utils/dag_processing.py", line 659, in _run_parsing_loop
self._processors.pop(processor.file_path)
KeyError: '/usr/local/lib/python3.8/site-packages/airflow/example_dags/example_bash_operator.py'
[2020-12-13 19:35:35,589] {dag_processing.py:396} WARNING - DagFileProcessorManager (PID=1029759) exited with exit code 1 - re-launching

# Fixed code
def _add_callback_to_queue(self, request: CallbackRequest):
    self._callback_to_execute[request.full_filepath].append(request)
    # Callback has a higher priority over DAG Run scheduling
    if request.full_filepath in self._file_path_queue:
        # Remove file paths matching request.full_filepath from self._file_path_queue
        # Since we are already going to use that filepath to run callback,
        # there is no need to have same file path again in the queue
        self._file_path_queue = [
            file_path
            for file_path in self._file_path_queue
            if file_path != request.full_filepath
        ]
    self._file_path_queue.insert(0, request.full_filepath)
# Buggy code
def dag_link(attr):
    """Generates a URL to the Graph View for a Dag."""
    dag_id = attr.get("dag_id")
    execution_date = attr.get("execution_date")
    url = url_for("Airflow.graph", dag_id=dag_id, execution_date=execution_date)
    return Markup('<a href="{}">{}</a>').format(url, dag_id)  # noqa


# Traceback (airflow.exceptions.SerializedDagNotFound)
Something bad has happened.
Please consider letting us know by creating a bug report using GitHub.

Python version: 3.6.12
Airflow version: 2.0.0
Node: 9097c882a712
-------------------------------------------------------------------------------
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.6/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/home/airflow/.local/lib/python3.6/site-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/airflow/.local/lib/python3.6/site-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/airflow/.local/lib/python3.6/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/home/airflow/.local/lib/python3.6/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/home/airflow/.local/lib/python3.6/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/www/auth.py", line 34, in decorated
return func(*args, **kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/www/decorators.py", line 97, in view_func
return f(*args, **kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/www/decorators.py", line 60, in wrapper
return f(*args, **kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/utils/session.py", line 65, in wrapper
return func(*args, session=session, **kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/www/views.py", line 2028, in graph
dag = current_app.dag_bag.get_dag(dag_id)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/utils/session.py", line 65, in wrapper
return func(*args, session=session, **kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/models/dagbag.py", line 171, in get_dag
self._add_dag_from_db(dag_id=dag_id, session=session)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/models/dagbag.py", line 227, in _add_dag_from_db
raise SerializedDagNotFound(f"DAG '{dag_id}' not found in serialized_dag table")
airflow.exceptions.SerializedDagNotFound: DAG 'None' not found in serialized_dag table

# Fixed code
def dag_link(attr):
    """Generates a URL to the Graph View for a Dag."""
    dag_id = attr.get("dag_id")
    execution_date = attr.get("execution_date")
    url = url_for("Airflow.graph", dag_id=dag_id, execution_date=execution_date)
    return (
        Markup('<a href="{}">{}</a>').format(url, dag_id) if dag_id else Markup("None")
    )  # noqa
# Buggy code
def __init__(
    self,
    *,
    bucket_name: str,
    prefix: str,
    aws_conn_id: str = "aws_default",
    verify: Optional[Union[bool, str]] = None,
    inactivity_period: float = 60 * 60,
    min_objects: int = 1,
    previous_objects: Optional[Set[str]] = None,
    allow_delete: bool = True,
    **kwargs,
) -> None:
    super().__init__(**kwargs)

    self.bucket = bucket_name
    self.prefix = prefix
    if inactivity_period < 0:
        raise ValueError("inactivity_period must be non-negative")
    self.inactivity_period = inactivity_period
    self.min_objects = min_objects
    self.previous_objects = previous_objects or set()
    self.inactivity_seconds = 0
    self.allow_delete = allow_delete
    self.aws_conn_id = aws_conn_id
    self.verify = verify
    self.last_activity_time: Optional[datetime] = None


# Traceback (AttributeError)
Traceback (most recent call last):
File "/usr/local/lib/airflow/airflow/models/taskinstance.py", line 966, in _run_raw_task
self.render_templates(context=context)
File "/usr/local/lib/airflow/airflow/models/taskinstance.py", line 1428, in render_templates
self.task.render_template_fields(context)
File "/usr/local/lib/airflow/airflow/models/baseoperator.py", line 706, in render_template_fields
self._do_render_template_fields(self, self.template_fields, context, jinja_env, set())
File "/usr/local/lib/airflow/airflow/models/baseoperator.py", line 711, in _do_render_template_fields
content = getattr(parent, attr_name)
AttributeError: 'S3KeysUnchangedSensor' object has no attribute 'bucket_name'

# Fixed code
def __init__(
    self,
    *,
    bucket_name: str,
    prefix: str,
    aws_conn_id: str = "aws_default",
    verify: Optional[Union[bool, str]] = None,
    inactivity_period: float = 60 * 60,
    min_objects: int = 1,
    previous_objects: Optional[Set[str]] = None,
    allow_delete: bool = True,
    **kwargs,
) -> None:
    super().__init__(**kwargs)

    self.bucket_name = bucket_name
    self.prefix = prefix
    if inactivity_period < 0:
        raise ValueError("inactivity_period must be non-negative")
    self.inactivity_period = inactivity_period
    self.min_objects = min_objects
    self.previous_objects = previous_objects or set()
    self.inactivity_seconds = 0
    self.allow_delete = allow_delete
    self.aws_conn_id = aws_conn_id
    self.verify = verify
    self.last_activity_time: Optional[datetime] = None
# Buggy code
def poke(self, context):
    return self.is_keys_unchanged(
        set(self.hook.list_keys(self.bucket, prefix=self.prefix))
    )


# Traceback (AttributeError)
Traceback (most recent call last):
File "/usr/local/lib/airflow/airflow/models/taskinstance.py", line 966, in _run_raw_task
self.render_templates(context=context)
File "/usr/local/lib/airflow/airflow/models/taskinstance.py", line 1428, in render_templates
self.task.render_template_fields(context)
File "/usr/local/lib/airflow/airflow/models/baseoperator.py", line 706, in render_template_fields
self._do_render_template_fields(self, self.template_fields, context, jinja_env, set())
File "/usr/local/lib/airflow/airflow/models/baseoperator.py", line 711, in _do_render_template_fields
content = getattr(parent, attr_name)
AttributeError: 'S3KeysUnchangedSensor' object has no attribute 'bucket_name'

# Fixed code
def poke(self, context):
    return self.is_keys_unchanged(
        set(self.hook.list_keys(self.bucket_name, prefix=self.prefix))
    )
# Buggy code
def read(self, filenames, encoding=None):
    super().read(filenames=filenames, encoding=encoding)
    self._validate()


# Traceback (NameError)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.6/site-packages/airflow/__init__.py", line 34, in <module>
from airflow import settings
File "/usr/local/lib/python3.6/site-packages/airflow/settings.py", line 35, in <module>
from airflow.configuration import AIRFLOW_HOME, WEBSERVER_CONFIG, conf  # NOQA F401
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 786, in <module>
conf.read(AIRFLOW_CONFIG)
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 447, in read
self._validate()
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 196, in _validate
self._validate_config_dependencies()
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 224, in _validate_config_dependencies
is_sqlite = "sqlite" in self.get('core', 'sql_alchemy_conn')
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 324, in get
option = self._get_option_from_secrets(deprecated_key, deprecated_section, key, section)
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 342, in _get_option_from_secrets
option = self._get_secret_option(section, key)
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 303, in _get_secret_option
return _get_config_value_from_secret_backend(secrets_path)
NameError: name '_get_config_value_from_secret_backend' is not defined

# Fixed code
def read(self, filenames, encoding=None):
    super().read(filenames=filenames, encoding=encoding)
# Buggy code
def read_dict(self, dictionary, source="<dict>"):
    super().read_dict(dictionary=dictionary, source=source)
    self._validate()


# Traceback (NameError)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.6/site-packages/airflow/__init__.py", line 34, in <module>
from airflow import settings
File "/usr/local/lib/python3.6/site-packages/airflow/settings.py", line 35, in <module>
from airflow.configuration import AIRFLOW_HOME, WEBSERVER_CONFIG, conf  # NOQA F401
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 786, in <module>
conf.read(AIRFLOW_CONFIG)
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 447, in read
self._validate()
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 196, in _validate
self._validate_config_dependencies()
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 224, in _validate_config_dependencies
is_sqlite = "sqlite" in self.get('core', 'sql_alchemy_conn')
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 324, in get
option = self._get_option_from_secrets(deprecated_key, deprecated_section, key, section)
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 342, in _get_option_from_secrets
option = self._get_secret_option(section, key)
File "/usr/local/lib/python3.6/site-packages/airflow/configuration.py", line 303, in _get_secret_option
return _get_config_value_from_secret_backend(secrets_path)
NameError: name '_get_config_value_from_secret_backend' is not defined

# Fixed code
def read_dict(self, dictionary, source="<dict>"):
    super().read_dict(dictionary=dictionary, source=source)
# Buggy code
def __init__(
    self,
    *,
    python_callable: Callable,
    task_id: str,
    op_args: Tuple[Any],
    op_kwargs: Dict[str, Any],
    multiple_outputs: bool = False,
    **kwargs,
) -> None:
    kwargs["task_id"] = self._get_unique_task_id(task_id, kwargs.get("dag"))
    super().__init__(**kwargs)
    self.python_callable = python_callable

    # Check that arguments can be binded
    signature(python_callable).bind(*op_args, **op_kwargs)
    self.multiple_outputs = multiple_outputs
    self.op_args = op_args
    self.op_kwargs = op_kwargs


# Traceback (airflow.exceptions.DuplicateTaskIdFound)
Broken DAG: [/files/dags/test.py] Traceback (most recent call last):
File "/opt/airflow/airflow/models/baseoperator.py", line 410, in __init__
task_group.add(self)
File "/opt/airflow/airflow/utils/task_group.py", line 140, in add
raise DuplicateTaskIdFound(f"Task id '{key}' has already been added to the DAG")
airflow.exceptions.DuplicateTaskIdFound: Task id 'show_tasks.show' has already been added to the DAG

# Fixed code
def __init__(
    self,
    *,
    python_callable: Callable,
    task_id: str,
    op_args: Tuple[Any],
    op_kwargs: Dict[str, Any],
    multiple_outputs: bool = False,
    **kwargs,
) -> None:
    kwargs["task_id"] = self._get_unique_task_id(
        task_id, kwargs.get("dag"), kwargs.get("task_group")
    )
    super().__init__(**kwargs)
    self.python_callable = python_callable

    # Check that arguments can be binded
    signature(python_callable).bind(*op_args, **op_kwargs)
    self.multiple_outputs = multiple_outputs
    self.op_args = op_args
    self.op_kwargs = op_kwargs
# Buggy code
def _get_unique_task_id(task_id: str, dag: Optional[DAG] = None) -> str:
    """
    Generate unique task id given a DAG (or if run in a DAG context)
    Ids are generated by appending a unique number to the end of
    the original task id.

    Example:
      task_id
      task_id__1
      task_id__2
      ...
      task_id__20
    """
    dag = dag or DagContext.get_current_dag()
    if not dag or task_id not in dag.task_ids:
        return task_id
    core = re.split(r"__\d+$", task_id)[0]
    suffixes = sorted(
        [
            int(re.split(r"^.+__", task_id)[1])
            for task_id in dag.task_ids
            if re.match(rf"^{core}__\d+$", task_id)
        ]
    )
    if not suffixes:
        return f"{core}__1"
    return f"{core}__{suffixes[-1] + 1}"


# Traceback (airflow.exceptions.DuplicateTaskIdFound)
Broken DAG: [/files/dags/test.py] Traceback (most recent call last):
File "/opt/airflow/airflow/models/baseoperator.py", line 410, in __init__
task_group.add(self)
File "/opt/airflow/airflow/utils/task_group.py", line 140, in add
raise DuplicateTaskIdFound(f"Task id '{key}' has already been added to the DAG")
airflow.exceptions.DuplicateTaskIdFound: Task id 'show_tasks.show' has already been added to the DAG

# Fixed code
def _get_unique_task_id(
    task_id: str, dag: Optional[DAG] = None, task_group: Optional[TaskGroup] = None
) -> str:
    """
    Generate unique task id given a DAG (or if run in a DAG context)
    Ids are generated by appending a unique number to the end of
    the original task id.

    Example:
      task_id
      task_id__1
      task_id__2
      ...
      task_id__20
    """
    dag = dag or DagContext.get_current_dag()
    if not dag:
        return task_id

    # We need to check if we are in the context of TaskGroup as the task_id may
    # already be altered
    task_group = task_group or TaskGroupContext.get_current_task_group(dag)
    tg_task_id = task_group.child_id(task_id) if task_group else task_id

    if tg_task_id not in dag.task_ids:
        return task_id
    core = re.split(r"__\d+$", task_id)[0]
    suffixes = sorted(
        [
            int(re.split(r"^.+__", task_id)[1])
            for task_id in dag.task_ids
            if re.match(rf"^{core}__\d+$", task_id)
        ]
    )
    if not suffixes:
        return f"{core}__1"
    return f"{core}__{suffixes[-1] + 1}"