# Buggy code
def replace_job(self, job, newjob, recursive=True):
    """Replace given job with new job."""
    add_to_targetjobs = job in self.targetjobs

    depending = list(self.depending[job].items())
    if self.finished(job):
        self._finished.add(newjob)

    self.delete_job(job, recursive=recursive)

    if add_to_targetjobs:
        self.targetjobs.add(newjob)

    self.cache_job(newjob)

    self.update([newjob])

    logger.debug("Replace {} with dynamic branch {}".format(job, newjob))
    for job_, files in depending:
        # if not job_.dynamic_input:
        logger.debug("updating depending job {}".format(job_))
        self.dependencies[job_][newjob].update(files)
        self.depending[newjob][job_].update(files)


# Traceback (FileNotFoundError)
Building DAG of jobs...
Using shell: /usr/bin/bash
Provided cores: 4
Rules claiming more threads will be scaled down.
Job counts:
count	jobs
1	all
1	corpus
2	shard
4

[Wed Dec 16 11:46:00 2020]
checkpoint shard:
output: data/batches.es
jobid: 2
wildcards: lang=es
Downstream jobs will be updated after completion.


[Wed Dec 16 11:46:00 2020]
checkpoint shard:
output: data/batches.en
jobid: 3
wildcards: lang=en
Downstream jobs will be updated after completion.

Updating job 1 (corpus).
Updating job 0 (all).
[Wed Dec 16 11:46:36 2020]
Finished job 2.
1 of 4 steps (25%) done

[Wed Dec 16 11:46:36 2020]
checkpoint shard:
output: data/batches.en
jobid: 3
wildcards: lang=en
Downstream jobs will be updated after completion.

Updating job 4 (corpus).
Updating job 5 (all).
[Wed Dec 16 11:47:00 2020]
Finished job 3.
2 of 40 steps (5%) done
Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/__init__.py", line 687, in snakemake
success = workflow.execute(
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/workflow.py", line 1006, in execute
success = scheduler.schedule()
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 470, in schedule
run = self.job_selector(needrun)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 764, in job_selector_greedy
c = list(map(self.job_reward, jobs))  # job rewards
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 847, in job_reward
input_size = job.inputsize
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/jobs.py", line 378, in inputsize
self._inputsize = sum(f.size for f in self.input)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/jobs.py", line 378, in <genexpr>
self._inputsize = sum(f.size for f in self.input)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 242, in wrapper
return func(self, *args, **kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 257, in wrapper
return func(self, *args, **kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 556, in size
return self.size_local
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 561, in size_local
self.check_broken_symlink()
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/io.py", line 566, in check_broken_symlink
if not self.exists_local and os.lstat(self.file):
FileNotFoundError: [Errno 2] No such file or directory: 'data/corpus.txt'
[Wed Dec 16 11:47:36 2020]
Error in rule shard:
jobid: 3
output: data/batches.en
shell:

for i in $(seq 0 $(( 2 % 5 + 1))); do # 0 .. 3 -> 4
for j in $(seq 0 $(( 1 % 2 + 1))); do # 0 .. 2 -> 3
mkdir -p data/en/$i/$j
echo 'hello en' > data/en/$i/$j/text
if [[ "en" == "es" ]]; then
sleep 3
else
sleep 5
fi
done
done
ls -d data/en/*/* > data/batches.en

(one of the commands exited with non-zero exit code; note that snakemake uses bash strict mode!)

Removing output files of failed job shard since they might be corrupted:
data/batches.en
exception calling callback for <Future at 0x7f745430c3a0 state=finished raised RuleException>
Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2318, in run_wrapper
run(
File "/home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile", line 37, in __rule_shard
checkpoints.shard.get(lang=trg_lang).output[0].open() as trg_f:
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/shell.py", line 202, in __new__
del cls._processes[jobid]
KeyError: 3

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 560, in _callback
raise ex
File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 546, in cached_or_run
run_func(*args)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2349, in run_wrapper
raise RuleException(
snakemake.exceptions.RuleException: KeyError in line 11 of /home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile:
3
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 2318, in run_wrapper
File "/home/cristian/Documentos/bitextor/files/tmp/min_snakemake_5_30_2/Snakefile", line 11, in __rule_shard

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 328, in _invoke_callbacks
callback(self)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/executors/__init__.py", line 572, in _callback
error_callback(job)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 579, in _error
self._handle_error(job)
File "/home/cristian/Documentos/bitextor/files/tmp/env_snake_performance_snakemake_fix_checkpoints/lib/python3.8/site-packages/snakemake/scheduler.py", line 589, in _handle_error
self.running.remove(job)
KeyError: shard

# Fixed code
def replace_job(self, job, newjob, recursive=True):
    """Replace given job with new job."""
    add_to_targetjobs = job in self.targetjobs
    jobid = self.jobid(job)

    depending = list(self.depending[job].items())
    if self.finished(job):
        self._finished.add(newjob)

    self.delete_job(job, recursive=recursive)
    self._jobid[newjob] = jobid

    if add_to_targetjobs:
        self.targetjobs.add(newjob)

    self.cache_job(newjob)

    self.update([newjob])

    logger.debug("Replace {} with dynamic branch {}".format(job, newjob))
    for job_, files in depending:
        # if not job_.dynamic_input:
        logger.debug("updating depending job {}".format(job_))
        self.dependencies[job_][newjob].update(files)
        self.depending[newjob][job_].update(files)