# Buggy code
def update_ready(self, jobs=None):
    """Update information whether a job is ready to execute.

    Given jobs must be needrun jobs!
    """

    if jobs is None:
        jobs = self.needrun_jobs

    potential_new_ready_jobs = False
    candidate_groups = set()
    for job in jobs:
        if job in self._ready_jobs:
            # job has been seen before, no need to process again
            continue
        if not self.finished(job) and self._ready(job):
            potential_new_ready_jobs = True
            if job.group is None:
                self._ready_jobs.add(job)
            else:
                group = self._group[job]
                group.finalize()
                candidate_groups.add(group)

    self._ready_jobs.update(
        group for group in candidate_groups if all(self._ready(job) for job in group)
    )
    return potential_new_ready_jobs


# Traceback (FileNotFoundError)
Building DAG of jobs...
Using shell: /usr/bin/bash
Provided cores: 4
Rules claiming more threads will be scaled down.
Job counts:
count	jobs
1	all
1	corpus
2	shard
4

[Wed Dec 16 11:46:00 2020]
checkpoint shard:
output: data/batches.es
jobid: 2
wildcards: lang=es
Downstream jobs will be updated after completion.


[Wed Dec 16 11:46:00 2020]
checkpoint shard:
output: data/batches.en
jobid: 3
wildcards: lang=en
Downstream jobs will be updated after completion.

Updating job 1 (corpus).
Updating job 0 (all).
[Wed Dec 16 11:46:36 2020]
Finished job 2.
1 of 4 steps (25%) done

[Wed Dec 16 11:46:36 2020]
checkpoint shard:
output: data/batches.en
jobid: 3
wildcards: lang=en
Downstream jobs will be updated after completion.

Updating job 4 (corpus).
Updating job 5 (all).
[Wed Dec 16 11:47:00 2020]
Finished job 3.
2 of 40 steps (5%) done
Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/__init__.py", line 687, in snakemake
success = workflow.execute(
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/workflow.py", line 1006, in execute
success = scheduler.schedule()
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 470, in schedule
run = self.job_selector(needrun)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 764, in job_selector_greedy
c = list(map(self.job_reward, jobs))  # job rewards
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 847, in job_reward
input_size = job.inputsize
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/jobs.py", line 378, in inputsize
self._inputsize = sum(f.size for f in self.input)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/jobs.py", line 378, in <genexpr>
self._inputsize = sum(f.size for f in self.input)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 242, in wrapper
return func(self, *args, **kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 257, in wrapper
return func(self, *args, **kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 556, in size
return self.size_local
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 561, in size_local
self.check_broken_symlink()
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 566, in check_broken_symlink
if not self.exists_local and os.lstat(self.file):
FileNotFoundError: [Errno 2] No such file or directory: 'data/corpus.txt'
[Wed Dec 16 11:47:36 2020]
Error in rule shard:
jobid: 3
output: data/batches.en
shell:

for i in $(seq 0 $(( 2 % 5 + 1))); do # 0 .. 3 -> 4
for j in $(seq 0 $(( 1 % 2 + 1))); do # 0 .. 2 -> 3
mkdir -p data/en/$i/$j
echo 'hello en' > data/en/$i/$j/text
if [[ "en" == "es" ]]; then
sleep 3
else
sleep 5
fi
done
done
ls -d data/en/*/* > data/batches.en

(one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)

Removing output files of failed job shard since they might be corrupted:
data/batches.en
exception calling callback for <Future at 0x7f745430c3a0 state=finished raised RuleException>
Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2318, in run_wrapper
run(
File "/home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile", line 37, in __rule_shard
checkpoints.shard.get(lang=trg_lang).output[0].open() as trg_f:
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/shell.py", line 202, in __new__
del cls._processes[jobid]
KeyError: 3

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 560, in _callback
raise ex
File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 546, in cached_or_run
run_func(*args)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2349, in run_wrapper
raise RuleException(
snakemake.exceptions.RuleException: KeyError in line 11 of /home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile:
3
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2318, in run_wrapper
File "/home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile", line 11, in __rule_shard

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 328, in _invoke_callbacks
callback(self)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 572, in _callback
error_callback(job)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 579, in _error
self._handle_error(job)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 589, in _handle_error
self.running.remove(job)
KeyError: shard

# Fixed code
def update_ready(self, jobs=None):
    """Update information whether a job is ready to execute.

    Given jobs must be needrun jobs!
    """

    if jobs is None:
        jobs = self.needrun_jobs

    potential_new_ready_jobs = False
    candidate_groups = set()
    for job in jobs:
        if job in self._ready_jobs or job in self._running:
            # job has been seen before or is running, no need to process again
            continue
        if not self.finished(job) and self._ready(job):
            potential_new_ready_jobs = True
            if job.group is None:
                self._ready_jobs.add(job)
            else:
                group = self._group[job]
                group.finalize()
                candidate_groups.add(group)

    self._ready_jobs.update(
        group for group in candidate_groups if all(self._ready(job) for job in group)
    )
    return potential_new_ready_jobs