# Buggy code
def __init__(
    self,
    workflow,
    rules=None,
    dryrun=False,
    targetfiles=None,
    targetrules=None,
    forceall=False,
    forcerules=None,
    forcefiles=None,
    priorityfiles=None,
    priorityrules=None,
    untilfiles=None,
    untilrules=None,
    omitfiles=None,
    omitrules=None,
    ignore_ambiguity=False,
    force_incomplete=False,
    ignore_incomplete=False,
    notemp=False,
    keep_remote_local=False,
    batch=None,
):
    self.dryrun = dryrun
    self.dependencies = defaultdict(partial(defaultdict, set))
    self.depending = defaultdict(partial(defaultdict, set))
    self._needrun = set()
    self._priority = dict()
    self._reason = defaultdict(Reason)
    self._finished = set()
    self._dynamic = set()
    self._len = 0
    self.workflow = workflow
    self.rules = set(rules)
    self.ignore_ambiguity = ignore_ambiguity
    self.targetfiles = targetfiles
    self.targetrules = targetrules
    self.priorityfiles = priorityfiles
    self.priorityrules = priorityrules
    self.targetjobs = set()
    self.prioritytargetjobs = set()
    self._ready_jobs = set()
    self.notemp = notemp
    self.keep_remote_local = keep_remote_local
    self._jobid = dict()
    self.job_cache = dict()
    self.conda_envs = dict()
    self.container_imgs = dict()
    self._progress = 0
    self._group = dict()
    self._n_until_ready = defaultdict(int)

    self.job_factory = JobFactory()
    self.group_job_factory = GroupJobFactory()

    self.forcerules = set()
    self.forcefiles = set()
    self.untilrules = set()
    self.untilfiles = set()
    self.omitrules = set()
    self.omitfiles = set()
    self.updated_subworkflow_files = set()
    if forceall:
        self.forcerules.update(self.rules)
    elif forcerules:
        self.forcerules.update(forcerules)
    if forcefiles:
        self.forcefiles.update(forcefiles)
    if untilrules:
        self.untilrules.update(set(rule.name for rule in untilrules))
    if untilfiles:
        self.untilfiles.update(untilfiles)
    if omitrules:
        self.omitrules.update(set(rule.name for rule in omitrules))
    if omitfiles:
        self.omitfiles.update(omitfiles)

    self.has_dynamic_rules = any(rule.dynamic_output for rule in self.rules)

    self.omitforce = set()

    self.batch = batch
    if batch is not None and not batch.is_final:
        # Since not all input files of a batching rule are considered, we cannot run
        # beyond that rule.
        # For the final batch, we do not need to omit anything.
        self.omitrules.add(batch.rulename)

    self.force_incomplete = force_incomplete
    self.ignore_incomplete = ignore_incomplete

    self.periodic_wildcard_detector = PeriodicityDetector()

    self.update_output_index()


# Traceback (FileNotFoundError)
Building DAG of jobs...
Using shell: /usr/bin/bash
Provided cores: 4
Rules claiming more threads will be scaled down.
Job counts:
count	jobs
1	all
1	corpus
2	shard
4

[Wed Dec 16 11:46:00 2020]
checkpoint shard:
output: data/batches.es
jobid: 2
wildcards: lang=es
Downstream jobs will be updated after completion.


[Wed Dec 16 11:46:00 2020]
checkpoint shard:
output: data/batches.en
jobid: 3
wildcards: lang=en
Downstream jobs will be updated after completion.

Updating job 1 (corpus).
Updating job 0 (all).
[Wed Dec 16 11:46:36 2020]
Finished job 2.
1 of 4 steps (25%) done

[Wed Dec 16 11:46:36 2020]
checkpoint shard:
output: data/batches.en
jobid: 3
wildcards: lang=en
Downstream jobs will be updated after completion.

Updating job 4 (corpus).
Updating job 5 (all).
[Wed Dec 16 11:47:00 2020]
Finished job 3.
2 of 40 steps (5%) done
Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/__init__.py", line 687, in snakemake
success = workflow.execute(
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/workflow.py", line 1006, in execute
success = scheduler.schedule()
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 470, in schedule
run = self.job_selector(needrun)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 764, in job_selector_greedy
c = list(map(self.job_reward, jobs))  # job rewards
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 847, in job_reward
input_size = job.inputsize
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/jobs.py", line 378, in inputsize
self._inputsize = sum(f.size for f in self.input)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/jobs.py", line 378, in <genexpr>
self._inputsize = sum(f.size for f in self.input)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 242, in wrapper
return func(self, *args, **kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 257, in wrapper
return func(self, *args, **kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 556, in size
return self.size_local
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 561, in size_local
self.check_broken_symlink()
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 566, in check_broken_symlink
if not self.exists_local and os.lstat(self.file):
FileNotFoundError: [Errno 2] No such file or directory: 'data/corpus.txt'
[Wed Dec 16 11:47:36 2020]
Error in rule shard:
jobid: 3
output: data/batches.en
shell:

for i in $(seq 0 $(( 2 % 5 + 1))); do # 0 .. 3 -> 4
for j in $(seq 0 $(( 1 % 2 + 1))); do # 0 .. 2 -> 3
mkdir -p data/en/$i/$j
echo 'hello en' > data/en/$i/$j/text
if [[ "en" == "es" ]]; then
sleep 3
else
sleep 5
fi
done
done
ls -d data/en/*/* > data/batches.en

(one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)

Removing output files of failed job shard since they might be corrupted:
data/batches.en
exception calling callback for <Future at 0x7f745430c3a0 state=finished raised RuleException>
Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2318, in run_wrapper
run(
File "/home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile", line 37, in __rule_shard
checkpoints.shard.get(lang=trg_lang).output[0].open() as trg_f:
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/shell.py", line 202, in __new__
del cls._processes[jobid]
KeyError: 3

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 560, in _callback
raise ex
File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 546, in cached_or_run
run_func(*args)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2349, in run_wrapper
raise RuleException(
snakemake.exceptions.RuleException: KeyError in line 11 of /home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile:
3
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2318, in run_wrapper
File "/home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile", line 11, in __rule_shard

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 328, in _invoke_callbacks
callback(self)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 572, in _callback
error_callback(job)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 579, in _error
self._handle_error(job)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 589, in _handle_error
self.running.remove(job)
KeyError: shard

# Fixed code
def __init__(
    self,
    workflow,
    rules=None,
    dryrun=False,
    targetfiles=None,
    targetrules=None,
    forceall=False,
    forcerules=None,
    forcefiles=None,
    priorityfiles=None,
    priorityrules=None,
    untilfiles=None,
    untilrules=None,
    omitfiles=None,
    omitrules=None,
    ignore_ambiguity=False,
    force_incomplete=False,
    ignore_incomplete=False,
    notemp=False,
    keep_remote_local=False,
    batch=None,
):
    self.dryrun = dryrun
    self.dependencies = defaultdict(partial(defaultdict, set))
    self.depending = defaultdict(partial(defaultdict, set))
    self._needrun = set()
    self._priority = dict()
    self._reason = defaultdict(Reason)
    self._finished = set()
    self._dynamic = set()
    self._len = 0
    self.workflow = workflow
    self.rules = set(rules)
    self.ignore_ambiguity = ignore_ambiguity
    self.targetfiles = targetfiles
    self.targetrules = targetrules
    self.priorityfiles = priorityfiles
    self.priorityrules = priorityrules
    self.targetjobs = set()
    self.prioritytargetjobs = set()
    self._ready_jobs = set()
    self.notemp = notemp
    self.keep_remote_local = keep_remote_local
    self._jobid = dict()
    self.job_cache = dict()
    self.conda_envs = dict()
    self.container_imgs = dict()
    self._progress = 0
    self._group = dict()
    self._n_until_ready = defaultdict(int)
    self._running = set()

    self.job_factory = JobFactory()
    self.group_job_factory = GroupJobFactory()

    self.forcerules = set()
    self.forcefiles = set()
    self.untilrules = set()
    self.untilfiles = set()
    self.omitrules = set()
    self.omitfiles = set()
    self.updated_subworkflow_files = set()
    if forceall:
        self.forcerules.update(self.rules)
    elif forcerules:
        self.forcerules.update(forcerules)
    if forcefiles:
        self.forcefiles.update(forcefiles)
    if untilrules:
        self.untilrules.update(set(rule.name for rule in untilrules))
    if untilfiles:
        self.untilfiles.update(untilfiles)
    if omitrules:
        self.omitrules.update(set(rule.name for rule in omitrules))
    if omitfiles:
        self.omitfiles.update(omitfiles)

    self.has_dynamic_rules = any(rule.dynamic_output for rule in self.rules)

    self.omitforce = set()

    self.batch = batch
    if batch is not None and not batch.is_final:
        # Since not all input files of a batching rule are considered, we cannot run
        # beyond that rule.
        # For the final batch, we do not need to omit anything.
        self.omitrules.add(batch.rulename)

    self.force_incomplete = force_incomplete
    self.ignore_incomplete = ignore_incomplete

    self.periodic_wildcard_detector = PeriodicityDetector()

    self.update_output_index()