# Buggy code
def finish(self, job, update_dynamic=True):
    """Finish a given job (e.g. remove from ready jobs, mark depending jobs
    as ready)."""

    # turn off this job's Reason
    self.reason(job).mark_finished()

    try:
        self._ready_jobs.remove(job)
    except KeyError:
        pass

    if job.is_group():
        jobs = job
    else:
        jobs = [job]

    self._finished.update(jobs)

    updated_dag = False
    if update_dynamic:
        updated_dag = self.update_checkpoint_dependencies(jobs)

    # mark depending jobs as ready
    # skip jobs that are marked as until jobs
    depending = [
        j
        for job in jobs
        for j in self.depending[job]
        if not self.in_until(job) and self.needrun(j)
    ]
    for job in depending:
        self._n_until_ready[job] -= 1

    potential_new_ready_jobs = self.update_ready(depending)

    for job in jobs:
        if update_dynamic and job.dynamic_output:
            logger.info("Dynamically updating jobs")
            newjob = self.update_dynamic(job)
            if newjob:
                # simulate that this job ran and was finished before
                self.omitforce.add(newjob)
                self._needrun.add(newjob)
                self._finished.add(newjob)
                updated_dag = True

                self.postprocess()
                self.handle_protected(newjob)
                self.handle_touch(newjob)

    if updated_dag:
        # We might have new jobs, so we need to ensure that all conda envs
        # and singularity images are set up.
        if self.workflow.use_singularity:
            self.pull_container_imgs()
        if self.workflow.use_conda:
            self.create_conda_envs()
        potential_new_ready_jobs = True

    return potential_new_ready_jobs


# Traceback (FileNotFoundError)
Building DAG of jobs...
Using shell: /usr/bin/bash
Provided cores: 4
Rules claiming more threads will be scaled down.
Job counts:
count	jobs
1	all
1	corpus
2	shard
4

[Wed Dec 16 11:46:00 2020]
checkpoint shard:
output: data/batches.es
jobid: 2
wildcards: lang=es
Downstream jobs will be updated after completion.


[Wed Dec 16 11:46:00 2020]
checkpoint shard:
output: data/batches.en
jobid: 3
wildcards: lang=en
Downstream jobs will be updated after completion.

Updating job 1 (corpus).
Updating job 0 (all).
[Wed Dec 16 11:46:36 2020]
Finished job 2.
1 of 4 steps (25%) done

[Wed Dec 16 11:46:36 2020]
checkpoint shard:
output: data/batches.en
jobid: 3
wildcards: lang=en
Downstream jobs will be updated after completion.

Updating job 4 (corpus).
Updating job 5 (all).
[Wed Dec 16 11:47:00 2020]
Finished job 3.
2 of 40 steps (5%) done
Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/__init__.py", line 687, in snakemake
success = workflow.execute(
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/workflow.py", line 1006, in execute
success = scheduler.schedule()
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 470, in schedule
run = self.job_selector(needrun)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 764, in job_selector_greedy
c = list(map(self.job_reward, jobs))  # job rewards
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 847, in job_reward
input_size = job.inputsize
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/jobs.py", line 378, in inputsize
self._inputsize = sum(f.size for f in self.input)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/jobs.py", line 378, in <genexpr>
self._inputsize = sum(f.size for f in self.input)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 242, in wrapper
return func(self, *args, **kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 257, in wrapper
return func(self, *args, **kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 556, in size
return self.size_local
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 561, in size_local
self.check_broken_symlink()
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 566, in check_broken_symlink
if not self.exists_local and os.lstat(self.file):
FileNotFoundError: [Errno 2] No such file or directory: 'data/corpus.txt'
[Wed Dec 16 11:47:36 2020]
Error in rule shard:
jobid: 3
output: data/batches.en
shell:

for i in $(seq 0 $(( 2 % 5 + 1))); do # 0 .. 3 -> 4
for j in $(seq 0 $(( 1 % 2 + 1))); do # 0 .. 2 -> 3
mkdir -p data/en/$i/$j
echo 'hello en' > data/en/$i/$j/text
if [[ "en" == "es" ]]; then
sleep 3
else
sleep 5
fi
done
done
ls -d data/en/*/* > data/batches.en

(one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)

Removing output files of failed job shard since they might be corrupted:
data/batches.en
exception calling callback for <Future at 0x7f745430c3a0 state=finished raised RuleException>
Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2318, in run_wrapper
run(
File "/home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile", line 37, in __rule_shard
checkpoints.shard.get(lang=trg_lang).output[0].open() as trg_f:
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/shell.py", line 202, in __new__
del cls._processes[jobid]
KeyError: 3

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 560, in _callback
raise ex
File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 546, in cached_or_run
run_func(*args)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2349, in run_wrapper
raise RuleException(
snakemake.exceptions.RuleException: KeyError in line 11 of /home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile:
3
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2318, in run_wrapper
File "/home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile", line 11, in __rule_shard

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 328, in _invoke_callbacks
callback(self)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 572, in _callback
error_callback(job)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 579, in _error
self._handle_error(job)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 589, in _handle_error
self.running.remove(job)
KeyError: shard

# Fixed code
def finish(self, job, update_dynamic=True):
    """Finish a given job (e.g. remove from ready jobs, mark depending jobs
    as ready)."""

    self._running.remove(job)

    # turn off this job's Reason
    self.reason(job).mark_finished()

    try:
        self._ready_jobs.remove(job)
    except KeyError:
        pass

    if job.is_group():
        jobs = job
    else:
        jobs = [job]

    self._finished.update(jobs)

    updated_dag = False
    if update_dynamic:
        updated_dag = self.update_checkpoint_dependencies(jobs)

    depending = [
        j
        for job in jobs
        for j in self.depending[job]
        if not self.in_until(job) and self.needrun(j)
    ]

    if not updated_dag:
        # Mark depending jobs as ready.
        # Skip jobs that are marked as until jobs.
        # This is not necessary if the DAG has been fully updated above.
        for job in depending:
            self._n_until_ready[job] -= 1

    potential_new_ready_jobs = self.update_ready(depending)

    for job in jobs:
        if update_dynamic and job.dynamic_output:
            logger.info("Dynamically updating jobs")
            newjob = self.update_dynamic(job)
            if newjob:
                # simulate that this job ran and was finished before
                self.omitforce.add(newjob)
                self._needrun.add(newjob)
                self._finished.add(newjob)
                updated_dag = True

                self.postprocess()
                self.handle_protected(newjob)
                self.handle_touch(newjob)

    if updated_dag:
        # We might have new jobs, so we need to ensure that all conda envs
        # and singularity images are set up.
        if self.workflow.use_singularity:
            self.pull_container_imgs()
        if self.workflow.use_conda:
            self.create_conda_envs()
        potential_new_ready_jobs = True

    return potential_new_ready_jobs