Repository: rajewsky-lab/spacemake (language: python)

Issue:
`name` added to config.yaml when creating a new run mode
Leaving this here so we don't forget. For example, running `spacemake config add_run_mode --name newrunmode` will add the following to `config.yaml`:

```
  newrunmode:
    name: newrunmode
    spatial_barcode_min_matches: 0
```

Not sure if it matters downstream though, i.e. if the `name` is used somewhere. To be updated.


Discussion / hints:

Fix patch:
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 00000000..b065e6eb
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,4 @@
+[run]
+concurrency = multiprocessing
+parallel = true
+sigterm = true
diff --git a/docs/conf.py b/docs/conf.py
index a2b30898..57c87b7e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -6,7 +6,7 @@
 copyright = '2021-2024, Rajewsky Lab'
 author = 'Tamas Ryszard Sztanka-Toth, Marvin Jens, Nikos Karaiskos, Nikolaus Rajewsky'
 
-version = '0.7.9'
+version = '0.8.0'
 release = version
 
 # -- General configuration
diff --git a/environment.yaml b/environment.yaml
index d2e77eed..f798d939 100644
--- a/environment.yaml
+++ b/environment.yaml
@@ -27,7 +27,12 @@ dependencies:
     - openjdk==11.0.15
     - pigz
     - pip:
-        - pandas>=1.3.0,<=1.5.1
+        - setproctitle
+        - isal
+        - pytest
+        - pytest-cov
+        - mrfifo>=0.3.0
+        - pandas>2
         - scanpy>=1.8.1
         - leidenalg>=0.8.1
         - numpy>=1.18.1
@@ -39,3 +44,6 @@ dependencies:
         - squidpy>=1.0.0
         - novosparc
         - opencv-python
+        - jinja2>=3.1.3
+        - matplotlib==3.8.4
+#        - pytest-optional-tests
diff --git a/setup.cfg b/setup.cfg
index ab27f686..021e058b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = spacemake
-version = 0.7.9
+version = attr: spacemake.contrib.__version__
 author = Tamas Ryszard Sztanka-Toth, Marvin Jens, Nikos Karaiskos, Nikolaus Rajewsky 
 author_email = TamasRyszard.Sztanka-Toth@mdc-berlin.de
 description = A bioinformatic pipeline for the analysis of spatial transcriptomic data
@@ -17,7 +17,7 @@ license = GPL
 
 [options]
 zip_safe = False
-python_requires = >=3.6
+python_requires = >=3.8
 include_package_data = True
 package_dir =
     spacemake = spacemake
@@ -30,15 +30,19 @@ spacemake =
     snakemake/scripts/*.Rmd
     snakemake/scripts/*.py
     data/*.csv
+    data/*.fa
     config/*.yaml
     longread/*.py
 
-# [options.data_files]
-# snakemake = spacemake/snakemake/dropseq.smk
-
 [options.entry_points]
 console_scripts = 
     alnstats = spacemake.alnstats:cmdline
     preprocess = spacemake.preprocess:cmdline
-    spacemake = spacemake.smk:cmdline
+    spacemake = spacemake.cmdline:cmdline
     pb_annotate = spacemake.longread.cmdline:cmdline
+
+[tool:pytest]
+testpaths = tests
+markers = 
+    big_download: needs to download large-ish files
+addopts = --cov=spacemake --cov-report html
diff --git a/spacemake/__init__.py b/spacemake/__init__.py
index b1cac2e9..7373d999 100644
--- a/spacemake/__init__.py
+++ b/spacemake/__init__.py
@@ -1,6 +1,6 @@
-__version__ = 1.0
-import matplotlib._path
-from . import preprocess as pp
-from . import spatial as sp
+# __version__ = 1.0
+# import matplotlib._path
+# from . import preprocess as pp
+# from . import spatial as sp
 
-from .smk import Spacemake
+# from .smk import Spacemake
diff --git a/spacemake/bin/BamTagHistogram.py b/spacemake/bin/BamTagHistogram.py
new file mode 100644
index 00000000..13d4581e
--- /dev/null
+++ b/spacemake/bin/BamTagHistogram.py
@@ -0,0 +1,183 @@
+import mrfifo as mf
+import logging
+
+
+def parse_args():
+    from spacemake.util import make_minimal_parser
+
+    parser = make_minimal_parser("BamTagHistogram")
+
+    parser.add_argument("--parallel", type=int, default=8)
+    parser.add_argument("--input", default="/dev/stdin")
+    parser.add_argument("--output", default="/dev/stdout")
+    parser.add_argument(
+        "--prefix-size",
+        default=4,
+        type=int,
+        help=(
+            "how many letters of the tag value are used to split the stream. "
+            "default=4 allows for up to (alphabet_size)^4 distinct parallel workers. "
+            "will be spread across workers by mod <args.parallel>"
+        ),
+    )
+    parser.add_argument("--prefix-alphabet", default="ACGTN")
+    parser.add_argument("--min-count", default=10, type=int)
+    parser.add_argument(
+        "--sort-mem",
+        default=8,
+        type=int,
+        help="how many GB are allowed to be used for sorting (default=8)",
+    )
+    parser.add_argument(
+        "--tag", default="CB", help="which BAM tag to count (default='CB')"
+    )
+
+    return parser.parse_args()
+
+
+def CB_distributor(
+    input, outputs, tag="CB", prefix_size=3, prefix_alphabet="ACGTN", n=8, **kw
+):
+    "ensure that the FIFOs are not managed"
+    assert type(input) is str
+    logger = logging.getLogger("mrfifo.parts.CB_distributor")
+    logger.debug(
+        f"reading from {input}, writing to {outputs} "
+        f"tag={tag} prefix_size={prefix_size} prefix_alphabet={prefix_alphabet} "
+        f"kw={kw}"
+    )
+
+    lkup = {}
+    from itertools import product
+
+    i = 0
+    for letters in product(*([prefix_alphabet] * prefix_size)):
+        prefix = "".join(letters).encode("ascii")
+        lkup[prefix] = i % n
+        i += 1
+
+    # for k, v in sorted(lkup.items()):
+    #     print(f"{k}\t{v}")
+
+    from mrfifo.fast_loops import distribute_by_substr
+
+    tag_lead = b"\t" + tag.encode("ascii") + b":Z:"
+    logger.debug(
+        f"scanning for tag-lead {tag_lead} and using next {prefix_size} bytes as prefix"
+    )
+    res = distribute_by_substr(
+        fin_name=input,
+        fifo_names=outputs,
+        sub_lookup=lkup,
+        sub_size=prefix_size,
+        sub_lead=tag_lead,
+        # **kw,
+    )
+    logger.debug("distribution complete")
+    return res
+
+
+def tag_counter(input, output, tag="CB", min_count=10):
+    from collections import defaultdict
+
+    counter = defaultdict(int)
+    stats = defaultdict(int)
+    import re
+
+    pattern = re.compile(f"{tag}:Z:(\S+)")
+    for sam_line in input:
+        stats["n_records"] += 1
+        flags = int(sam_line.split("\t")[1])
+        if flags & 256:
+            # 'not primary alignment' bit is set
+            stats["n_secondary"] += 1
+            continue
+
+        if m := re.search(pattern, sam_line):
+            stats["n_tagged"] += 1
+            tag_val = m.groups(0)[0]
+            counter[tag_val] += 1
+
+    stats["n_values"] = len(counter)
+    for value, count in counter.items():
+        if count >= min_count:
+            stats["n_above_cut"] += 1
+            output.write(f"{count}\t{value}\n")
+
+    return stats
+
+
+def sort_function(input, output, n=8, sort_mem_gigs=8, header=None):
+    import os
+
+    if header is None:
+        header = rf"# INPUT={args.input} TAG={args.tag} FILTER_PCR_DUPLICATES=false READ_QUALITY=0\n"
+
+    if output.endswith(".gz"):
+        cmd = (
+            f'{{ printf "{header}"; sort -rnk 1 -S {sort_mem_gigs}G --parallel={n} {input}; }}'
+            f"| python -m isal.igzip -c > {output}"
+        )
+    else:
+        cmd = f'{{ printf "{header}"; sort -rnk 1 -S {sort_mem_gigs}G --parallel={n} {input}; }} > {output}'
+
+    import subprocess
+
+    subprocess.call(cmd, shell=True)
+
+
+def main(args):
+    w = (
+        mf.Workflow("BamTagHistogram")
+        .BAM_reader(
+            input=args.input,
+            mode="S",
+            threads=4,
+        )
+        .distribute(
+            input=mf.FIFO("input_sam", "rt"),
+            outputs=mf.FIFO("dist_{n}", "wt", n=args.parallel),
+            func=CB_distributor,
+            tag=args.tag,
+            prefix_size=args.prefix_size,
+            prefix_alphabet=args.prefix_alphabet,
+            n=args.parallel,
+        )
+        .workers(
+            func=tag_counter,
+            tag=args.tag,
+            input=mf.FIFO("dist_{n}", "rt"),
+            output=mf.FIFO("counts_{n}", "wt"),
+            n=args.parallel,
+            min_count=args.min_count,
+        )
+        .collect(
+            inputs=mf.FIFO("counts_{n}", "rt", n=args.parallel),
+            output=mf.FIFO("unsorted", "wt"),
+            chunk_size=1,
+        )
+        .funnel(
+            input=mf.FIFO("unsorted", "rt"),
+            output=args.output,
+            func=sort_function,
+            _manage_fifos=False,
+        )
+        .run()
+    )
+    stats = mf.util.CountDict()
+    for jobname, d in w.result_dict.items():
+        if "worker" in jobname:
+            stats.add_other_stats(d)
+
+    df = stats.get_stats_df()
+    df["input"] = args.input
+    print(df.set_index("input"))
+    return w
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    import spacemake.util as util
+
+    util.setup_logging(args)
+    main(args)
diff --git a/spacemake/bin/fastq_to_uBAM.py b/spacemake/bin/fastq_to_uBAM.py
new file mode 100644
index 00000000..1f5f7d16
--- /dev/null
+++ b/spacemake/bin/fastq_to_uBAM.py
@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+from spacemake.contrib import __version__, __author__, __license__, __email__
+import logging
+import os
+import sys
+import numpy as np
+import multiprocessing as mp
+from collections import defaultdict
+from time import time
+import spacemake.util as util
+import mrfifo as mf
+
+# TODO:
+# * load params from YAML (code from opseq)
+# * save params to YAML in run-folder to document
+# * refactor into multiple modules?
+
+format_func_template = """
+import re
+
+def format_func(qname=None, r2_qname=None, r2_qual=None, r1=None, r2=None, R1=None, R2=None, **kw):
+    #qparts = r2_qname.split(':N:0:')
+    #if len(qparts) > 1:
+    #    i5i7 = qparts[1].replace('+', '')
+    #else:
+    #    i5i7 = None
+
+    cell = {cell}
+
+    attrs = dict(
+        cell=cell,
+        raw=cell,
+        UMI={UMI},
+        seq={seq},
+        qual={qual},
+        fqid=r2_qname,
+    )
+
+    return attrs
+"""
+
+
+def safety_check_eval(s, danger="();."):
+    chars = set(list(s))
+    if chars & set(list(danger)):
+        return False
+    else:
+        return True
+
+
+def make_formatter_from_args(args):
+    if not args.disable_safety:
+        assert safety_check_eval(args.cell)
+        assert safety_check_eval(args.UMI)
+        assert safety_check_eval(args.seq)
+        assert safety_check_eval(args.qual)
+
+    # This trickery here compiles a python function that
+    # embeds the user-specified code for cell, UMI
+    # we replace globals() with an empty dictionary d,
+    d = {}
+    exec(
+        format_func_template.format(
+            cell=args.cell, UMI=args.UMI, seq=args.seq, qual=args.qual
+        ),
+        d,
+    )
+    # from said dict we can now retrieve the function object
+    # and call it many times over
+    func = d["format_func"]
+    return func
+
+
+def make_sam_record(
+    fqid,
+    seq,
+    qual,
+    flag=4,
+    tags=[("CB", "{cell}"), ("MI", "{UMI}"), ("CR", "{raw}"), ("RG", "A")],
+    **kw,
+):
+    tag_str = "\t".join([f"{tag}:Z:{tstr.format(**kw)}" for tag, tstr in tags])
+    return f"{fqid}\t{flag}\t*\t0\t0\t*\t*\t0\t0\t{seq}\t{qual}\t{tag_str}\n"
+
+
+def quality_trim(fq_src, min_qual=20, phred_base=33):
+    for name, seq, qual in fq_src:
+        end = len(seq)
+        q = np.array(bytearray(qual.encode("ASCII"))) - phred_base
+        qtrim = q >= min_qual
+        new_end = end - (qtrim[::-1]).argmax()
+
+        # TODO: yield A3,T3 adapter-trimming tags
+        # TODO: convert to cutadapt/BWA qual-trim logic
+        if new_end != end:
+            qual = qual[:new_end]
+            seq = seq[:new_end]
+
+        yield (name, seq, qual)
+
+
+def render_to_sam(fq1, fq2, sam_out, args, **kwargs):
+    logger = util.setup_logging(args, "spacemake.bin.fastq_to_uBAM.worker")
+    logger.debug(
+        f"starting up with fq1={fq1}, fq2={fq2} sam_out={sam_out} and args={args}"
+    )
+
+    def iter_paired(fq1, fq2):
+        fq_src1 = util.FASTQ_src(fq1)
+        fq_src2 = util.FASTQ_src(fq2)
+        if args.min_qual_trim:
+            fq_src2 = quality_trim(
+                fq_src2, min_qual=args.min_qual_trim, phred_base=args.phred_base
+            )
+
+        return zip(fq_src1, fq_src2)
+
+    def iter_single(fq2):
+        fq_src2 = util.FASTQ_src(fq2)
+        if args.min_qual_trim:
+            fq_src2 = quality_trim(
+                fq_src2, min_qual=args.min_qual_trim, phred_base=args.phred_base
+            )
+
+        for fqid, seq, qual in fq_src2:
+            yield (fqid, "NA", "NA"), (fqid, seq, qual)
+
+    if fq1:
+        ingress = iter_paired(fq1, fq2)
+    else:
+        ingress = iter_single(fq2)
+
+    # TODO:
+    # decide how we want to handle multiple input files
+    # somehow I prefer sequential processing over complicated state-change
+    # question is, how do we keep the output sam from being closed?
+    # ideally, insert a serializer there and provide a way to join() some processes before starting new
+    # ones... (needs thinking)
+    # kw = vars(args)
+    # # kw.update(par)
+    # import argparse
+    # args = argparse.Namespace(**kw)
+
+    N = mf.util.CountDict()
+
+    fmt = make_formatter_from_args(args)  # , **params
+
+    for (fqid, r1, q1), (_, r2, q2) in ingress:
+        N.count("total")
+        attrs = fmt(r2_qname=fqid, r1=r1, r1_qual=q1, r2=r2, r2_qual=q2)
+        sam_out.write(make_sam_record(flag=4, **attrs))
+
+    return N
+    # if args.paired_end:
+    #             # if args.paired_end[0] == 'r':
+    #             #     # rev_comp R1 and reverse qual1
+    #             #     q1 = q1[::-1]
+    #             #     r1 = util.rev_comp(r1)
+
+    #             # if args.paired_end[1] == 'r':
+    #             #     # rev_comp R2 and reverse qual2
+    #             #     q2 = q2[::-1]
+    #             #     r2 = util.rev_comp(r2)
+
+    #             rec1 = fmt.make_bam_record(
+    #                 qname=fqid,
+    #                 r1="THISSHOULDNEVERBEREFERENCED",
+    #                 r2=r1,
+    #                 R1=r1,
+    #                 R2=r2,
+    #                 r2_qual=q1,
+    #                 r2_qname=fqid,
+    #                 flag=69,  # unmapped, paired, first in pair
+    #             )
+    #             rec2 = fmt.make_bam_record(
+    #                 qname=fqid,
+    #                 r1="THISSHOULDNEVERBEREFERENCED",
+    #                 r2=r2,
+    #                 R1=r1,
+    #                 R2=r2,
+    #                 r2_qual=q2,
+    #                 r2_qname=fqid,
+    #                 flag=133,  # unmapped, paired, second in pair
+    #             )
+    #             results.append(rec1)
+    #             results.append(rec2)
+
+    #         else:
+
+
+def main(args):
+    input_reads1, input_reads2, input_params = get_input_params(args)
+    have_read1 = set([str(r1) != "None" for r1 in input_reads1]) == set([True])
+
+    # queues for communication between processes
+    w = (
+        mf.Workflow("fastq_to_uBAM")
+        # open reads2.fastq.gz
+        .gz_reader(inputs=input_reads2, output=mf.FIFO("read2", "wb")).distribute(
+            input=mf.FIFO("read2", "rt"),
+            outputs=mf.FIFO("r2_{n}", "wt", n=args.parallel),
+            chunk_size=args.chunk_size * 4,
+        )
+    )
+
+    if have_read1:
+        # open reads1.fastq.gz
+        w.gz_reader(inputs=input_reads1, output=mf.FIFO("read1", "wb"))
+        w.distribute(
+            input=mf.FIFO("read1", "rt"),
+            outputs=mf.FIFO("r1_{n}", "wt", n=args.parallel),
+            chunk_size=args.chunk_size * 4,
+        )
+        # process in parallel workers
+        w.workers(
+            func=render_to_sam,
+            fq1=mf.FIFO("r1_{n}", "rt"),
+            fq2=mf.FIFO("r2_{n}", "rt"),
+            sam_out=mf.FIFO("sam_{n}", "wt"),
+            args=args,
+            n=args.parallel,
+        )
+    else:
+        # process in parallel workers
+        w.workers(
+            func=render_to_sam,
+            fq1=None,
+            fq2=mf.FIFO("r2_{n}", "rt"),
+            sam_out=mf.FIFO("sam_{n}", "wt"),
+            args=args,
+            n=args.parallel,
+        )
+
+    # combine output streams
+    w.collect(
+        inputs=mf.FIFO("sam_{n}", "rt", n=args.parallel),
+        chunk_size=args.chunk_size,
+        custom_header=mf.util.make_SAM_header(
+            prog_id="fastq_to_uBAM",
+            prog_name="fastq_to_uBAM.py",
+            prog_version=__version__,
+            rg_name=args.sample,
+        ),
+        # output="/dev/stdout"
+        # )
+        output=mf.FIFO("sam_combined", "wt"),
+    )
+    # compress to BAM
+    w.funnel(
+        func=mf.parts.bam_writer,  # mf.parts.null_writer, #
+        input=mf.FIFO("sam_combined", "rt"),
+        output=args.out_bam,
+        _manage_fifos=False,
+        fmt="Sbh",
+        threads=16,
+    )
+    return w.run()
+
+
+def get_input_params(args):
+    import pandas as pd
+    import csv
+
+    if str(args.matrix) != "None":
+        df = pd.read_csv(args.matrix, sep=",", index_col=None, quoting=csv.QUOTE_NONE)
+        if not "R1" in df.columns:
+            df["R1"] = "None"
+
+        R1 = df["R1"].to_list()
+        R2 = df["R2"].to_list()
+        # TODO extract other column values into a list of dictionaries (most importantly cell=...)
+        # which can override how the formatter in the workers processes the raw reads
+        if "cell" in df.columns:
+            # params = [{'cell': f'"{c}"'} for c in df['cell']]
+            params = [{"cell": c} for c in df["cell"]]
+        else:
+            params = [
+                {},
+            ] * len(R1)
+
+    else:
+        R1 = [
+            args.read1,
+        ]
+        R2 = [
+            args.read2,
+        ]
+        params = [
+            {},
+        ]
+
+    return R1, R2, params
+
+
+def parse_args():
+    import spacemake.util as util
+
+    parser = util.make_minimal_parser(
+        "fastq_to_uBAM",
+        description="Convert raw reads1 and reads2 FASTQ into a single BAM file with cell barcode and UMI as BAM-tags",
+    )
+    parser.add_argument(
+        "--matrix",
+        default=None,
+        help="sample_matrix.csv file desribing from where to get read1 and read2 (FASTQ format)",
+    )
+    parser.add_argument(
+        "--read1",
+        default=None,
+        help="source from where to get read1 (FASTQ format)",
+    )
+    parser.add_argument(
+        "--read2",
+        default="/dev/stdin",
+        help="source from where to get read2 (FASTQ format)",
+        # required=True,
+    )
+    parser.add_argument("--cell", default="r1[8:20][::-1]")
+    parser.add_argument("--UMI", default="r1[0:8]")
+    parser.add_argument("--seq", default="r2")
+    parser.add_argument("--qual", default="r2_qual")
+    parser.add_argument("--disable-safety", default=False, type=bool)
+
+    parser.add_argument(
+        "--paired-end",
+        default=None,
+        choices=["fr", "ff", "rf", "rr"],
+        help="read1 and read2 are paired end mates and store both in the BAM",
+    )
+    parser.add_argument(
+        "--out-bam",
+        default="/dev/stdout",
+        help="output for unaligned BAM records (default=/dev/stdout) ",
+    )
+    parser.add_argument(
+        "--save-stats",
+        default="preprocessing_stats.txt",
+        help="store statistics in this file",
+    )
+    parser.add_argument(
+        "--save-cell-barcodes",
+        default="",
+        help="store (raw) cell barcode counts in this file. Numbers add up to number of total raw reads.",
+    )
+    parser.add_argument(
+        "--fq-qual",
+        default="E",
+        help="phred qual for assigned barcode bases in FASTQ output (default='E')",
+    )
+    parser.add_argument(
+        "--min-qual-trim",
+        default=0,
+        type=int,
+        help="clip low quality-bases from a read2 3' end (e.g. pre-detected adapter)",
+    )
+    parser.add_argument(
+        "--min-len",
+        default=18,
+        type=int,
+        help="minimum read2 length to keep after quality trimming (default=18)",
+    )
+    parser.add_argument(
+        "--phred-base",
+        default=33,
+        type=int,
+        help="phred quality base (default=33)",
+    )
+    parser.add_argument(
+        "--parallel", default=1, type=int, help="how many processes to spawn"
+    )
+    parser.add_argument(
+        "--chunk-size",
+        default=10,
+        type=int,
+        help="how many consecutive reads are assigned to the same worker (default=10)",
+    )
+    # SAM standard now supports CB=corrected cell barcode, CR=original cell barcode, and MI=molecule identifier/UMI
+    parser.add_argument(
+        "--bam-tags",
+        default="CR:{cell},CB:{cell},MI:{UMI},RG:A",
+        help="a template of comma-separated BAM tags to generate. Variables are replaced with extracted cell barcode, UMI etc.",
+    )
+    args = parser.parse_args()
+    if args.parallel < 1:
+        raise ValueError(f"--parallel {args.parallel} is invalid. Must be >= 1")
+
+    return args
+
+
+def cmdline():
+    args = parse_args()
+    util.setup_logging(args, name="spacemake.bin.fastq_to_uBAM")
+
+    if not args.read2:
+        raise ValueError("bam output requires --read2 parameter")
+
+    return main(args)
+
+
+if __name__ == "__main__":
+    res = cmdline()
diff --git a/spacemake/cmdline.py b/spacemake/cmdline.py
new file mode 100644
index 00000000..522c90ac
--- /dev/null
+++ b/spacemake/cmdline.py
@@ -0,0 +1,1081 @@
+import argparse
+import os
+import logging
+from spacemake.util import message_aggregation
+from spacemake.config import ConfigFile
+from spacemake.contrib import __version__, __license__, __author__, __email__
+
+logger_name = "spacemake.main"
+logger = logging.getLogger(logger_name)
+
+
+def get_project_sample_parser(allow_multiple=False, prepend="", help_extra=""):
+    """
+    Return a parser for project_id's and sample_id's
+
+    :param allow_multiple: if true, we allow multiple projects and samples,
+        and the parser will have a `--project_id_list` and a `--sample_id_list`
+        parameter
+    :param prepend: a string that will be prepended before the arguments
+    :param help_extra: extra help message
+    :return: a parser object
+    :rtype: argparse.ArgumentParser
+    """
+    logger.debug(f"get_project_sample_parser(prepend={prepend}) called")
+
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
+    project_argument = "project_id"
+    sample_argument = "sample_id"
+    required = True
+    nargs = None
+    default = None
+
+    if allow_multiple:
+        project_argument = f"{project_argument}_list"
+        sample_argument = f"{sample_argument}_list"
+        required = False
+        nargs = "*"
+        default = []
+
+    parser.add_argument(
+        f"--{prepend}{project_argument}",
+        type=str,
+        required=required,
+        nargs=nargs,
+        default=default,
+        help=f"{project_argument} {help_extra}",
+    )
+
+    parser.add_argument(
+        f"--{prepend}{sample_argument}",
+        type=str,
+        required=required,
+        nargs=nargs,
+        default=default,
+        help=f"{sample_argument} {help_extra}",
+    )
+
+    return parser
+
+
+def get_add_sample_sheet_parser():
+    """
+    Returns parser for sample sheet addition
+
+    :return: parser
+    :rtype: argparse.ArgumentParser
+    """
+    logger.debug("get_add_sample_sheet_parser() called")
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False,
+        description="add a new sample sheet to the samples",
+        add_help=False,
+    )
+
+    parser.add_argument(
+        "--sample_sheet",
+        type=str,
+        help="the path to the Illumina sample sheet",
+        required=True,
+    )
+    parser.add_argument(
+        "--basecalls_dir",
+        type=str,
+        help="path to the basecalls directory",
+        required=True,
+    )
+
+    return parser
+
+
+def get_sample_main_variables_parser(
+    defaults=False,
+    species_required=False,
+    main_variables=[
+        "barcode_flavor",
+        "adapter_flavor",
+        "species",
+        "puck",
+        "run_mode",
+        "map_strategy",
+    ],
+):
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
+    logger.debug(
+        f"get_sample_main_variables_parser called with main_variables={main_variables}"
+    )
+
+    if "barcode_flavor" in main_variables:
+        parser.add_argument(
+            "--barcode_flavor",
+            type=str,
+            default="dropseq" if defaults else None,
+            help="barcode flavor for this sample",
+        )
+
+    if "adapter_flavor" in main_variables:
+        parser.add_argument(
+            "--adapter_flavor",
+            type=str,
+            default="dropseq" if defaults else None,
+            help="barcode flavor for this sample",
+        )
+
+    if "species" in main_variables:
+        parser.add_argument(
+            "--species",
+            type=str,
+            help="add the name of the species for this sample",
+            required=species_required,
+        )
+
+    if "map_strategy" in main_variables:
+        parser.add_argument(
+            "--map_strategy",
+            type=str,
+            help="string constant definining mapping strategy. Can be multi-stage and use bowtie2 or STAR (see documentation)",
+            required=False,
+            default=None,
+        )
+
+    if "puck" in main_variables:
+        parser.add_argument(
+            "--puck",
+            type=str,
+            help="name of the puck for this sample. if puck_type contains a \n"
+            + "`barcodes` path to a coordinate file, those coordinates\n"
+            + " will be used when processing this sample. if \n"
+            + " not provided, a default puck will be used with \n"
+            + "width_um=3000, spot_diameter_um=10",
+        )
+
+        parser.add_argument(
+            "--puck_barcode_file_id",
+            type=str,
+            help="puck_barcode_file_id of the sample to be added/update",
+            nargs="+",
+        )
+
+    parser.add_argument(
+        "--puck_barcode_file",
+        type=str,
+        nargs="+",
+        help="the path to the file contining (x,y) positions of the barcodes",
+    )
+
+    if "run_mode" in main_variables:
+        parser.add_argument(
+            "--run_mode",
+            type=str,
+            nargs="+",
+            help="run_mode names for this sample.\n"
+            + "the sample will be processed using the provided run_modes.\n"
+            + "for merged samples, if left empty, the run_modes of the \n"
+            + "merged (input) samples will be intersected.\n",
+        )
+
+    return parser
+
+
+def get_sample_extra_info_parser():
+    logger.debug("get_sample_extra_info_parser() called")
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
+
+    parser.add_argument(
+        "--investigator",
+        type=str,
+        help="add the name of the investigator(s) responsible for this sample",
+    )
+
+    parser.add_argument("--experiment", type=str, help="description of the experiment")
+
+    import datetime
+
+    parser.add_argument(
+        "--sequencing_date",
+        type=datetime.date.fromisoformat,
+        help="sequencing date of the sample. format: YYYY-MM-DD",
+    )
+
+    return parser
+
+
+def get_data_parser(reads_required=False):
+    """
+    Returns a parser which contain extra arguments for a given sample.
+    The returned parser will contain the --R1, --R2, --longreads,
+    --longread-signature, --barcode_flavor, --species, --puck,  --puck_barcode_file_id,
+    --puck_barcode_file, --investigator, --experiment, --sequencing_date,
+    --run_mode arguments.
+
+    :param species_required: if true, the --species argument will be required
+        during parsing.
+    :param reads_required: if true, --R1, --R2, and --longreads arguments will be
+        required during parsing.
+    """
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
+    logger.debug("get_data_parser() called")
+    parser.add_argument(
+        "--R1",
+        type=str,
+        help=".fastq.gz file path to R1 reads",
+        nargs="+",
+        required=reads_required,
+    )
+
+    parser.add_argument(
+        "--R2",
+        type=str,
+        help=".fastq.gz file path to R2 reads",
+        nargs="+",
+        required=reads_required,
+    )
+
+    parser.add_argument(
+        "--reads",
+        type=str,
+        default="None",
+        help="path to a CSV file listing reads for the sample (can be used for pooling reads and also for aggregating bulk samples into one big analysis with different cell barcodes for each sample)",
+        required=reads_required,
+    )
+
+    parser.add_argument(
+        "--dge",
+        type=str,
+        help="Path to dge matrix. spacemake can also handle already processed"
+        + " digital expression data",
+        required=reads_required,
+    )
+
+    parser.add_argument(
+        "--longreads",
+        type=str,
+        help="fastq(.gz)|fq(.gz)|bam file path to pacbio long reads for library debugging",
+        required=reads_required,
+    )
+
+    parser.add_argument(
+        "--longread-signature",
+        type=str,
+        help="identify the expected longread signature (see longread.yaml)",
+    )
+
+    return parser
+
+
+def get_set_remove_variable_subparsers(
+    parent_parser, variable_name, func, prepend="", allow_multiple=False
+):
+    """get_set_remove_variable_subparsers.
+
+    :param parent_parser:
+    :param variable_name:
+    :param func:
+    :param prepend:
+    :param allow_multiple:
+    """
+    if allow_multiple:
+        nargs = "+"
+    else:
+        nargs = None
+
+    logger.debug(f"get_set_remove_variable_subparsers(varname={variable_name}) called")
+
+    def get_action_parser(action):
+        """get_action_parser.
+
+        :param action:
+        """
+        action_parser = parent_parser.add_parser(
+            f"{action}_{variable_name}",
+            parents=[get_project_sample_parser(allow_multiple=True)],
+            description=f"{action} {variable_name} for several projects/samples",
+            help=f"{action} {variable_name} for several projects/samples",
+        )
+        action_parser.add_argument(
+            f"--{variable_name}", type=str, required=True, nargs=nargs
+        )
+        action_parser.set_defaults(func=func, variable=variable_name, action=action)
+
+        return action_parser
+
+    set_parser = get_action_parser("set")
+
+    if allow_multiple:
+        set_parser.add_argument("--keep_old", action="store_true")
+
+        remove_parser = get_action_parser("remove")
+
+
+def get_action_sample_parser(parent_parser, action, func):
+    """get_action_sample_parser.
+
+    :param parent_parser:
+    :param action:
+    :param func:
+    """
+    logger.debug(f"get_action_sample_parser(action={action}) called")
+    if action not in ["add", "update", "delete", "merge"]:
+        raise ValueError(f"Invalid action: {action}")
+
+    if action == "merge":
+        parser_name = "merge_samples"
+        msg = "merge samples"
+        parents = [
+            get_project_sample_parser(
+                prepend="merged_", help_extra="of the newly created merged sample"
+            ),
+            get_project_sample_parser(
+                allow_multiple=True, help_extra="of the samples to be merged"
+            ),
+        ]
+    else:
+        parser_name = f"{action}_sample"
+        msg = f"{action} a sample"
+        parents = [get_project_sample_parser()]
+
+    if action == "add":
+        # add arguments for species, run_mode, barcode_flavor and puck
+        parents.append(
+            get_sample_main_variables_parser(
+                defaults=True,
+                species_required=True,
+            )
+        )
+        # add arguments for R1/R1, dge, longread
+        parents.append(get_data_parser())
+        # add arguments for extra sample info
+        parents.append(get_sample_extra_info_parser())
+    elif action == "update":
+        # add main variables parser
+        parents.append(get_sample_main_variables_parser())
+        # add arguments for R1/R1, dge, longread
+        parents.append(get_data_parser())
+        # add possibility to add extra info
+        parents.append(get_sample_extra_info_parser())
+    elif action == "merge":
+        # add main variables parser
+        # when merging, only let the user overwrite puck and run_mode
+        parents.append(
+            get_sample_main_variables_parser(
+                main_variables=["run_mode", "puck"],
+            )
+        )
+
+        # add possibility to add extra info
+        parents.append(get_sample_extra_info_parser())
+
+    sample_parser = parent_parser.add_parser(
+        parser_name, description=msg, help=msg, parents=parents
+    )
+    sample_parser.set_defaults(func=func, action=action)
+
+
+def setup_project_parser(parent_parser_subparsers):
+    """setup_project_parser.
+
+    :param attach_to:
+    """
+    parser = parent_parser_subparsers.add_parser(
+        "projects",
+        help="manage projects and samples",
+        description="Using one of the subcommands specified below, it is possible to"
+        + " add/update/remove projects and their settings",
+    )
+    subparsers = parser.add_subparsers()
+
+    help_desc = {
+        "add_sample_sheet": "add projects and samples from Illumina sample sheet",
+        "add_samples_from_yaml": "add several samples at once from a .yaml file",
+        "merge_samples": "merge several samples into one. "
+        + "samples need to have the same species",
+        "list": "list several project(s) and sample(s) and their settings",
+    }
+
+    # ADD/UPDATE/DELETE/MERGE sample(s)
+    # get parsers for adding, deleting and updating a sample
+    for action in ["add", "update", "delete"]:
+        get_action_sample_parser(
+            subparsers,
+            action,
+            # attach the function to the parser
+            func=add_update_delete_sample_cmdline,
+        )
+
+    # get parser for merging
+    get_action_sample_parser(subparsers, "merge", func=merge_samples_cmdline)
+
+    # LIST PROJECTS
+    # always show these variables
+    always_show = [
+        "puck_barcode_file_id",
+        "species",
+        "investigator",
+        "sequencing_date",
+        "experiment",
+        "run_mode",
+        "barcode_flavor",
+        "map_strategy",
+    ]
+    import spacemake.project_df as spdf
+
+    remaining_options = [
+        x
+        for x in spdf.ProjectDF.project_df_default_values.keys()
+        if x not in always_show
+    ]
+    list_projects = subparsers.add_parser(
+        "list",
+        description=help_desc["list"],
+        help=help_desc["list"],
+        parents=[
+            get_project_sample_parser(
+                allow_multiple=True,
+                help_extra="subset the data. If none provided, all will be displayed",
+            )
+        ],
+    )
+    list_projects.add_argument(
+        "--variables",
+        help="which extra variables to display per sample? "
+        + f"{always_show} will always be shown.",
+        choices=remaining_options,
+        default=[],
+        nargs="*",
+    )
+    list_projects.set_defaults(func=list_projects_cmdline, always_show=always_show)
+
+    # ADD SAMPLE SHEET
+    sample_add_sample_sheet = subparsers.add_parser(
+        "add_sample_sheet",
+        description=help_desc["add_sample_sheet"],
+        help=help_desc["add_sample_sheet"],
+        parents=[get_add_sample_sheet_parser()],
+    )
+    sample_add_sample_sheet.set_defaults(func=add_sample_sheet_cmdline)
+
+    # ADD SAMPLES FROM YAML
+    sample_add_samples_yaml = subparsers.add_parser(
+        "add_samples_from_yaml",
+        description=help_desc["add_samples_from_yaml"],
+        help=help_desc["add_samples_from_yaml"],
+    )
+    sample_add_samples_yaml.add_argument(
+        "--samples_yaml",
+        type=str,
+        required=True,
+        help="path to the .yaml file containing sample info",
+    )
+    sample_add_samples_yaml.set_defaults(func=add_samples_from_yaml_cmdline)
+
+    # get set/remove parser for each main variable
+    # this will add parser for:
+    # setting/removing run_mode
+    # setting species
+    # setting puck
+    # setting barcode_flavor
+    # NOTE: the ConfigFile.main_variable_sg2type keys determine
+    # which commandline args will be defined for the parser.
+    # this is a little un-intuitive...
+    # TODO: cleaner factory functions for the commandline-parsers
+    for main_var_sg, main_var_type in ConfigFile.main_variable_sg2type.items():
+        allow_multiple = False
+        if isinstance(main_var_type, str) and main_var_type.endswith("_list"):
+            allow_multiple = True
+
+        get_set_remove_variable_subparsers(
+            subparsers,
+            variable_name=main_var_sg,
+            func=set_remove_variable_cmdline,
+            allow_multiple=allow_multiple,
+        )
+    return parser
+
+
+def get_run_parser():
+    """get_run_parser."""
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
+
+    parser.add_argument(
+        "--cores", default=1, type=int, help="number of cores to be used in total"
+    )
+    parser.add_argument(
+        "--dryrun",
+        "-n",
+        action="store_true",
+        help="invokes a dry snakemake run, printing only commands",
+    )
+    parser.add_argument(
+        "--debug",
+        default="",
+        help=f"comma-separated list of logging-domains for which you want DEBUG output",
+    )    
+    parser.add_argument(
+        "--rerun-incomplete",
+        "--ri",
+        action="store_true",
+        help="forces snakemake to rerun incompletely generated files",
+    )
+    parser.add_argument(
+        "--keep-going",
+        action="store_true",
+        help="if a job fails, keep executing independent jobs",
+    )
+    parser.add_argument(
+        "--printshellcmds",
+        "-p",
+        action="store_true",
+        help="print shell commands for each rule, if exist",
+    )
+    parser.add_argument(
+        "--touch",
+        "-t",
+        action="store_true",
+        help="rather than running the rules, just touch each file",
+    )
+    parser.add_argument(
+        "--with_fastqc",
+        "-wfqc",
+        action="store_true",
+        help="Run also fastqc as part of the spacemake run",
+    )
+
+    return parser
+
+
+def setup_init_parser(parent_parser_subparsers):
+    """setup_init_parser.
+
+    :param parent_parser_subparsers:
+    """
+    parser_init = parent_parser_subparsers.add_parser(
+        "init",
+        help="initialise spacemake: create config files, download genomes and annotations",
+    )
+    parser_init.add_argument(
+        "--root_dir",
+        default="",
+        help="where to output the results of the spacemake run. defaults to .",
+    )
+    parser_init.add_argument(
+        "--temp_dir",
+        default="/tmp",
+        help="temporary directory used when running spacemake. defaults to /tmp",
+    )
+    parser_init.add_argument(
+        "--download_species",
+        default=False,
+        help="if set, upon initialisation, spacemake will download the mouse and human genome and index",
+        action="store_true",
+    )
+    parser_init.add_argument(
+        "--dropseq_tools",
+        help="absolute path to dropseq_tools directory",
+        required=True,
+    )    
+    parser_init.set_defaults(func=spacemake_init)
+
+    return parser_init
+
+
+def setup_run_parser(parent_parser_subparsers):
+    """setup_run_parser.
+
+    :param parent_parser_subparsers:
+    """
+    parser_run = parent_parser_subparsers.add_parser(
+        "run", help="run spacemake", parents=[get_run_parser()]
+    )
+    parser_run.set_defaults(func=spacemake_run)
+
+    # create subparsers
+    parser_run_subparsers = parser_run.add_subparsers()
+
+    downsampling_parser = parser_run_subparsers.add_parser(
+        "downsample",
+        help="run downsampling analysis for a list of projects/samples",
+        parents=[get_project_sample_parser(allow_multiple=True), get_run_parser()],
+    )
+    downsampling_parser.set_defaults(downsample=True, func=spacemake_run)
+
+    # parser_novosparc = novosparc_spacemake_parser(parser_run_subparsers)
+    # parser_novosparc.set_defaults(novosparc_reconstruct=True,
+    #    func=lambda args: spacemake_run(pdf, args))
+
+    return parser_run
+
+
+#####################################################
+# actual command-line functions, used as call-backs #
+#####################################################
+
+
+@message_aggregation(logger_name)
+def spacemake_init(args):
+    """spacemake_init.
+
+    :param args:
+    """
+    import snakemake
+    import yaml
+    from spacemake.snakemake.variables import config_path
+
+    if os.path.isfile(config_path):
+        msg = "spacemake has already been initiated in this directory.\n"
+        msg += "use other commands to run and analyse samples."
+        logger.info(msg)
+        return 0
+    initial_config = os.path.join(os.path.dirname(__file__), "data/config/config.yaml")
+
+    # initialise config file
+    cf = ConfigFile.from_yaml(initial_config)
+    # update the file path
+    cf.set_file_path(config_path)
+
+    # add variables from args
+    cf.variables["root_dir"] = args["root_dir"]
+    cf.variables["temp_dir"] = args["temp_dir"]
+
+    cf.variables['external_bin'] = {
+        'dropseq_tools' : args['dropseq_tools']
+    }
+    
+    cf.variables["microscopy_out"] = args.get("microscopy_out", "")
+
+    if args["download_species"]:
+        species_data_config_file = os.path.join(
+            os.path.dirname(__file__), "data/config/species_data_url.yaml"
+        )
+        species_data_config = yaml.load(
+            open(species_data_config_file), Loader=yaml.FullLoader
+        )
+
+        # add keys as species to config
+        species = list(species_data_config.keys())
+        snakemake_config = {"root_dir": ""}
+        snakemake_config["species"] = species
+
+        # define the pattern
+        snakemake_config[
+            "annotation_file_pattern"
+        ] = "species_data/{species}/{species}_{data_type}.gtf"
+        snakemake_config[
+            "genome_file_pattern"
+        ] = "species_data/{species}/{species}_{data_type}.fa"
+
+        # the to be saved file paths
+        species_info = {}
+        species_files_exist = []
+
+        for sp in species:
+            if not sp in species_info.keys():
+                species_info[sp] = {}
+            for data_type in ["genome", "annotation"]:
+                # save download url to be passed to snakemake
+                snakemake_config[sp + "_" + data_type + "_url"] = species_data_config[
+                    sp
+                ][data_type]
+                # save file path of the result
+                species_file = snakemake_config[data_type + "_file_pattern"].format(
+                    species=sp, data_type=data_type
+                )
+                species_info[sp][data_type] = species_file
+
+                # add bool if file exists
+                species_files_exist.append(os.path.isfile(species_file))
+
+        # if any of the files are missing
+        if not all(species_files_exist):
+            # get the snakefile for downloading the .gtf and .fa files
+            snakefile = os.path.join(
+                os.path.dirname(__file__), "snakemake/species_init.smk"
+            )
+            # run snakemake: download species data and place them in the right place
+            snakemake.snakemake(snakefile, cores=1, config=snakemake_config)
+
+        for key, value in species_info.items():
+            cf.add_variable(
+                "species", key, reference="genome", sequence=value["genome"], annotation=value["annotation"]
+            )
+
+    # copy visium_puck_barcode_file
+    dest_visium_path = "puck_data/visium_barcode_positions.csv"
+    logger.info(f"Moving visium barcodes to {dest_visium_path}")
+    os.makedirs(os.path.dirname(dest_visium_path), exist_ok=True)
+    from shutil import copyfile
+
+    copyfile(
+        os.path.join(os.path.dirname(__file__), "data/visium_barcode_positions.csv"),
+        dest_visium_path,
+    )
+
+    # copy openst_coordinate_system
+    dest_puck_collection_path = "puck_data/openst_coordinate_system.csv"
+    logger.info(f"Moving puck collection coordinate system to {dest_puck_collection_path}")
+    os.makedirs(os.path.dirname(dest_puck_collection_path), exist_ok=True)
+    copyfile(
+        os.path.join(os.path.dirname(__file__), "data/puck_collection/openst_coordinate_system.csv"),
+        dest_puck_collection_path,
+    )
+
+    # save
+    cf.dump()
+
+
+@message_aggregation(logger_name)
+def spacemake_run(args):
+    """spacemake_run.
+
+    :param args:
+    """
+    from spacemake.errors import SpacemakeError
+    import snakemake
+
+    import spacemake.snakemake.variables as var
+
+    if not os.path.isfile(var.config_path):
+        msg = "spacemake has not been initalised yet.\n"
+        msg += "please run `spacemake init` to start a new project"
+
+        raise SpacemakeError(msg)
+
+    from spacemake.project_df import get_global_ProjectDF
+
+    root = logging.getLogger("spacemake")
+    # TODO: Why is args a dictionary and not argparse.Namespace ?
+    if args['debug']:
+        # activate cmdline requested debug output for specific domains (comma-separated)
+        for logger_name in args['debug'].split(","):
+            if logger_name:
+                root.info(f"setting domain {logger_name} to DEBUG")
+                logging.getLogger(logger_name.replace("root", "")).setLevel(
+                    logging.DEBUG
+                )
+
+    pdf = get_global_ProjectDF()
+    samples = []
+    projects = []
+    targets = ["run_analysis"]
+    with_fastqc = args.get("with_fastqc", False)
+
+    downsample = args.get("downsample", False)
+    novosparc_reconstruct = args.get("novosparc_reconstruct", False)
+
+    if downsample:
+        targets = ["downsample"]
+        samples = args.get("sample_id_list", [])
+        projects = args.get("project_id_list", [])
+
+    if novosparc_reconstruct:
+        targets = ["novosparc"]
+        import spacemake.smk as smk
+
+        novosparc_variables = smk.get_novosparc_variables(pdf, args)
+    else:
+        novosparc_variables = {}
+
+    config_variables = {
+        "project_df": pdf.file_path,
+        "samples": samples,
+        "projects": projects,
+        "with_fastqc": with_fastqc,
+        "pwd": os.getcwd(),
+        "log_debug" : args['debug'],
+    }
+
+    # join config_variables and novosparc_variables
+    # to flatten the dictionary
+    config_variables = {**config_variables, **novosparc_variables}
+
+    # get the snakefile
+    snakefile = os.path.join(os.path.dirname(__file__), "snakemake/main.smk")
+    # run snakemake
+    preprocess_finished = snakemake.snakemake(
+        snakefile,
+        configfiles=[var.config_path],
+        cores=args["cores"],
+        dryrun=args["dryrun"],
+        targets=['get_stats_prealigned_barcodes', 'unload_genome_flag'],
+        touch=args["touch"],
+        force_incomplete=args["rerun_incomplete"],
+        keepgoing=args["keep_going"],
+        printshellcmds=args["printshellcmds"],
+        config=config_variables,
+    )
+
+    if preprocess_finished is False:
+        raise SpacemakeError("an error occurred while snakemake() ran")
+
+    # update valid pucks (above threshold) before continuing to downstream
+    # this performs counting of matching barcodes after alignment
+    pdf.update_project_df_barcode_matches(prealigned=True)
+    pdf.consolidate_pucks_merged_samples()
+    pdf.dump()
+
+    # whitelisting of barcodes
+    preprocess_finished = snakemake.snakemake(
+        snakefile,
+        configfiles=[var.config_path],
+        cores=args["cores"],
+        dryrun=args["dryrun"],
+        targets=['get_whitelist_barcodes'],
+        touch=args["touch"],
+        force_incomplete=args["rerun_incomplete"],
+        keepgoing=args["keep_going"],
+        printshellcmds=args["printshellcmds"],
+        config=config_variables,
+    )
+
+    pdf.update_project_df_barcode_matches()
+    pdf.dump()
+
+    # run snakemake quantification and reports
+    analysis_finished = snakemake.snakemake(
+        snakefile,
+        configfiles=[var.config_path],
+        cores=args["cores"],
+        dryrun=args["dryrun"],
+        targets=targets,
+        touch=args["touch"],
+        force_incomplete=args["rerun_incomplete"],
+        keepgoing=args["keep_going"],
+        printshellcmds=args["printshellcmds"],
+        config=config_variables,
+    )
+
+    if analysis_finished is False:
+        raise SpacemakeError("an error occurred while snakemake() ran")
+
+    # at the very end dump the project_data_frame
+    pdf.dump()
+
+
+@message_aggregation(logger_name)
+def add_update_delete_sample_cmdline(args):
+    """add_update_delete_sample_cmdline.
+
+    :param args:
+    """
+    action = args["action"]
+    from spacemake.project_df import get_global_ProjectDF
+
+    pdf = get_global_ProjectDF()
+    # remove the action from args
+    del args["action"]
+
+    if action == "add" or action == "update":
+        func = lambda **kwargs: pdf.add_update_sample(action=action, **kwargs)
+    elif action == "delete":
+        func = pdf.delete_sample
+
+    sample = func(**args)
+    pdf.dump()
+
+
+@message_aggregation(logger_name)
+def add_sample_sheet_cmdline(args):
+    """add_sample_sheet_cmdline.
+
+    :param args:
+    """
+    from spacemake.project_df import get_global_ProjectDF
+
+    pdf = get_global_ProjectDF()
+
+    pdf.add_sample_sheet(args["sample_sheet"], args["basecalls_dir"])
+
+    pdf.dump()
+
+
+@message_aggregation(logger_name)
+def add_samples_from_yaml_cmdline(args):
+    """add_samples_from_yaml_cmdline.
+
+    :param args:
+    """
+    from spacemake.project_df import get_global_ProjectDF
+
+    pdf = get_global_ProjectDF()
+    pdf.add_samples_from_yaml(args["samples_yaml"])
+
+    pdf.dump()
+
+
+@message_aggregation(logger_name)
+def set_remove_variable_cmdline(args):
+    """set_remove_variable_cmdline.
+
+    :param args:
+    """
+    variable_name = args["variable"]
+    action = args["action"]
+
+    variable_key = args[variable_name]
+    from spacemake.project_df import get_global_ProjectDF
+
+    pdf = get_global_ProjectDF()
+
+    pdf.set_remove_variable(
+        variable_name=variable_name,
+        variable_key=variable_key,
+        action=action,
+        project_id_list=args["project_id_list"],
+        sample_id_list=args["sample_id_list"],
+        keep_old=args.get("keep_old", False),
+    )
+
+    pdf.dump()
+
+
+@message_aggregation(logger_name)
+def merge_samples_cmdline(args):
+    """merge_samples_cmdline.
+
+    :param args:
+    """
+    from spacemake.project_df import get_global_ProjectDF
+
+    pdf = get_global_ProjectDF()
+    pdf.merge_samples(**args)
+
+    pdf.dump()
+
+
+@message_aggregation(logger_name)
+def list_projects_cmdline(args):
+    """list_projects_cmdline.
+
+    :param args:
+    """
+    from spacemake.project_df import get_global_ProjectDF
+
+    pdf = get_global_ProjectDF()
+
+    projects = args["project_id_list"]
+    samples = args["sample_id_list"]
+    variables = args["always_show"] + args["variables"]
+
+    df = pdf.df
+    logger = logging.getLogger(logger_name)
+
+    if projects != [] or samples != []:
+        df = df.query("project_id in @projects or sample_id in @samples")
+        logger.inof(f"listing projects: {projects} and samples: {samples}")
+    else:
+        logger.info("listing all projects and samples")
+
+    logger.info(f"variables used: {variables}")
+
+    # print the table
+    logger.info(df.loc[:, variables].__str__())
+
+
+def make_main_parser():
+    #################
+    # DEFINE PARSER #
+    #################
+    # we add main parser to a dictionary
+    # so that we can later call the help function
+    # based on the sub-command. this is to print the
+    # -h (help) functionality if no parameters are provided,
+    # rather than printing an error.
+    # import spacemake.smk as smk
+
+    parser_main = argparse.ArgumentParser(
+        allow_abbrev=False,
+        description="spacemake: bioinformatic pipeline for processing and analysis of spatial-transcriptomics data",
+    )
+
+    parser_main.add_argument("--version", action="store_true")
+
+    parser_main_subparsers = parser_main.add_subparsers(
+        help="sub-command help", dest="subcommand"
+    )
+
+    parser_run = None
+    parser_projects = None
+    parser_config = None
+    parser_init = None
+    parser_spatial = None
+
+    ##################
+    # SPACEMAKE INIT #
+    ##################
+    parser_init = setup_init_parser(parser_main_subparsers)
+
+    ####################
+    # SPACEMAKE CONFIG #
+    ####################
+    ## spacemake_config args
+    from spacemake.config import setup_config_parser
+    from spacemake.snakemake.variables import config_path
+
+    if os.path.isfile(config_path):
+        # save config file
+        parser_config = setup_config_parser(parser_main_subparsers)
+
+        ############################
+        # SPACEMAKE PROJECT/SAMPLE #
+        ############################
+        from spacemake.cmdline import setup_project_parser
+
+        parser_projects = setup_project_parser(parser_main_subparsers)
+
+        #################
+        # SPACEMAKE RUN #
+        #################
+        parser_run = setup_run_parser(parser_main_subparsers)
+
+        #####################
+        # SPACEMAKE SPATIAL #
+        #####################
+        from spacemake.spatial.cmdline import setup_spatial_parser
+        parser_spatial = setup_spatial_parser(parser_main_subparsers)
+
+    parser_dict = {
+        "init": parser_init,
+        "config": parser_config,
+        "projects": parser_projects,
+        "run": parser_run,
+        "main": parser_main,
+        "spatial": parser_spatial,
+    }
+
+    return parser_main, parser_dict
+
+
+def cmdline():
+    # import importlib.metadata
+    """cmdline."""
+    parser_main, parser_dict = make_main_parser()
+    args = parser_main.parse_args()
+
+    if args.version and args.subcommand is None:
+        import spacemake.contrib
+        print(spacemake.contrib.__version__)
+        return 0
+    else:
+        del args.version
+
+    # get the function to be run
+    if "func" in args:
+        func = args.func
+    # else print help
+    else:
+        if args.subcommand is not None:
+            parser_dict[args.subcommand].print_help()
+        else:
+            parser_dict["main"].print_help()
+        return 0
+
+    # get the args and delete the func key, get only set values
+    args = {key: value for key, value in vars(args).items() if value is not None}
+    args.pop("func", None)
+    # pop also main,
+    args.pop("subcommand", None)
+
+    return func(args)
+
+
+if __name__ == "__main__":
+    cmdline()
diff --git a/spacemake/config.py b/spacemake/config.py
index 955d033a..24595a61 100644
--- a/spacemake/config.py
+++ b/spacemake/config.py
@@ -1,5 +1,6 @@
 import os
-import yaml
+
+# import yaml
 import argparse
 import re
 import logging
@@ -11,6 +12,16 @@
 logger_name = "spacemake.config"
 
 
+__global_config = None
+
+
+def get_global_config(root="."):
+    global __global_config
+    if __global_config is None:
+        __global_config = ConfigFile.from_yaml(f"{root}/config.yaml")
+
+    return __global_config
+
 def get_puck_parser(required=True):
     parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
     parser.add_argument("--name", help="name of the puck", type=str, required=True)
@@ -40,7 +51,6 @@ def get_puck_parser(required=True):
 
     return parser
 
-
 def get_run_mode_parser(required=True):
     parser = argparse.ArgumentParser(
         allow_abbrev=False,
@@ -360,7 +370,7 @@ def add_update_delete_variable_cmdline(config, args):
 def list_variables_cmdline(config, args):
     variable = args["variable"]
     del args["variable"]
-
+    import yaml
     config.logger.info(f"Listing {variable}")
     config.logger.info(yaml.dump(config.variables[variable]))
 
@@ -449,8 +459,11 @@ class ConfigFile:
     main_variables_pl2sg = {
         "pucks": "puck",
         "barcode_flavors": "barcode_flavor",
+        "adapters": "adapter",
+        "adapter_flavors": "adapter_flavor",
         "run_modes": "run_mode",
         "species": "species",
+        "quant": "quant",
     }
 
     main_variables_sg2pl = {value: key for key, value in main_variables_pl2sg.items()}
@@ -458,6 +471,8 @@ class ConfigFile:
     main_variable_sg2type = {
         "puck": str,
         "barcode_flavor": str,
+        "adapter": str,
+        "adapter_flavor": str,
         "run_mode": "str_list",
         "species": str,
         "map_strategy": str,
@@ -465,19 +480,23 @@ class ConfigFile:
 
     def __init__(self):
         self.variables = {
-            "root_dir": "",
+            "root_dir": ".",
             "temp_dir": "/tmp",
             "species": {},
             "barcode_flavors": {},
+            "adapter_flavors": {},
+            "adapters": {},
             "run_modes": {},
             "pucks": {},
+            "quant": {},
         }
         self.file_path = "config.yaml"
         self.logger = logging.getLogger(logger_name)
 
     @classmethod
-    def from_yaml(cls, file_path):
+    def from_yaml(cls, file_path="config.yaml"):
         cf = cls()
+        import yaml
 
         config_yaml_variables = None
         with open(file_path, "r") as f:
@@ -493,7 +512,6 @@ def from_yaml(cls, file_path):
 
             # correct variables to ensure backward compatibility
             cf.correct()
-
             # check which variables do not exist, if they dont,
             # copy them from initial config
             for main_variable in cf.main_variables_pl2sg:
@@ -521,8 +539,31 @@ def from_yaml(cls, file_path):
                     default_val.update(cf.variables[var_with_default]["default"])
                     cf.variables[var_with_default]["default"] = default_val
 
+        cf.expand_strings()
         return cf
 
+    def expand_strings(self, **kw):
+        kw['spacemake_dir'] = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+        # print(f"expanding strings in config with context={kw}")
+        import collections.abc as abc
+        def recurse(d):
+            if type(d) == str:
+                try:
+                    return d.format(**kw)
+                except KeyError:
+                    return d
+
+            if isinstance(d, abc.Mapping):
+                for key, value in d.items():
+                    d[key] = recurse(value)
+
+            elif isinstance(d, abc.Collection):
+                d = [recurse(x) for x in d]
+
+            return d
+
+        self.variables = recurse(self.variables)
+
     def assert_main_variable(self, variable):
         if variable not in self.main_variables_pl2sg.keys():
             raise UnrecognisedConfigVariable(
@@ -627,6 +668,9 @@ def correct(self):
             self.variables["run_modes"][run_mode_name] = variables
 
     def dump(self):
+        import yaml
+
+        self.logger.debug(f"writing config.yaml to '{self.file_path}")
         with open(self.file_path, "w") as fo:
             fo.write(yaml.dump(self.variables))
 
@@ -673,7 +717,7 @@ def delete_variable(self, variable_name, variable_key, **kw):
 
         return variable_data
 
-    def process_run_mode_args(self, **kwargs):
+    def process_run_mode_args(self, name=None, **kwargs):
         # typeset boolean values of run_mode
         default_run_mode = self.get_variable("run_modes", "default")
 
@@ -683,28 +727,86 @@ def process_run_mode_args(self, **kwargs):
 
         return kwargs
 
-    def process_barcode_flavor_args(self, cell_barcode=None, umi=None, name=None):
+    def process_barcode_flavor_args(self, **kw):
         bam_tags = "CR:{cell},CB:{cell},MI:{UMI},RG:{assigned}"
-
+        
         # r(1|2) and then string slice
-        to_match = r"r(1|2)(\[((?=-)-\d+|\d)*\:((?=-)-\d+|\d*)(\:((?=-)-\d+|\d*))*\])+$"
+        # this is too restrictive. What about umi = "r1[12:20] + r2[:8]" ? 
+        # would be perfectly reasonable 
+        # to_match = r"r(1|2)(\[((?=-)-\d+|\d)*\:((?=-)-\d+|\d*)(\:((?=-)-\d+|\d*))*\])+$"
 
-        if umi is not None and re.match(to_match, umi) is None:
-            raise InvalidBarcodeStructure("umi", to_match)
+        UMI = kw.get("UMI", None)
+        # if umi is not None and re.match(to_match, umi) is None:
+        #     raise InvalidBarcodeStructureError("umi", to_match)
 
-        if cell_barcode is not None and re.match(to_match, cell_barcode) is None:
-            raise InvalidBarcodeStructure("umi", to_match)
+        cell_barcode = kw.get("cell", None)
+        # if cell_barcode is not None and re.match(to_match, cell_barcode) is None:
+        #     raise InvalidBarcodeStructureError("cell", to_match)
 
         barcode_flavor = {"bam_tags": bam_tags}
 
-        if umi is not None:
-            barcode_flavor["UMI"] = umi
+        if UMI is not None:
+            barcode_flavor["UMI"] = UMI
 
         if cell_barcode is not None:
             barcode_flavor["cell"] = cell_barcode
 
+        for key, value in kw.items():
+            if value is not None:
+                barcode_flavor[key] = value
+
         return barcode_flavor
 
+    def process_adapter_args(self, name=None, seq=None):
+        # print(f"processing {name} {seq}")
+        if (name is None) or (seq is None):
+            raise ValueError(f"need a name and a sequence: name={name} seq={seq}")
+
+        return {name: seq}
+
+    def process_adapter_flavor_args(self, cut_left=[], cut_right=[], name=None, **kw):
+        def parse_adapters(cut):
+            type_d = {
+                "max_errors": float,
+                "min_overlap": int,
+                "min_base_qual": float,
+            }
+            from collections import OrderedDict
+
+            out = []
+            for c in cut:
+                parts = c.split(":")
+                adap_name = parts[0]
+                if adap_name != "Q" and not adap_name in self.variables["adapters"]:
+                    raise KeyError(
+                        (
+                            f"unknown adapter {adap_name}! We have "
+                            f"{','.join(self.variables['adapters'])} "
+                            " you can add new adapter with --add_adapter "
+                        )
+                    )
+                d = {}
+                if len(parts) > 1:
+                    for param_str in parts[1:]:
+                        key, value = param_str.split("=")
+                        d[key] = type_d.get(key, str)(value)
+
+                out.append({adap_name: d})
+
+            return out
+
+        adapter_flavor = {
+            "cut_left": parse_adapters(cut_left),
+            "cut_right": parse_adapters(cut_right),
+        }
+
+        for key, value in kw.items():
+            if value is not None:
+                adapter_flavor[key] = value
+
+        # print(f"processed adapter_flavors {adapter_flavor}")
+        return {name: adapter_flavor}
+
     def process_species_args(
         self,
         name=None,
@@ -736,7 +838,7 @@ def process_species_args(
             d["BT2_flags"] = BT2_flags
 
         if STAR_flags:
-            d["STAR_flags"] = BT2_flags
+            d["STAR_flags"] = STAR_flags
 
         species_refs = self.variables["species"].get(name, {})
         species_refs[reference] = d
@@ -766,6 +868,10 @@ def process_puck_args(self, width_um=None, spot_diameter_um=None, barcodes=None,
     def process_variable_args(self, variable, **kwargs):
         if variable == "barcode_flavors":
             return self.process_barcode_flavor_args(**kwargs)
+        elif variable == "adapters":
+            return self.process_adapter_args(**kwargs)
+        elif variable == "adapter_flavors":
+            return self.process_adapter_flavor_args(**kwargs)
         elif variable == "run_modes":
             return self.process_run_mode_args(**kwargs)
         elif variable == "pucks":
@@ -797,7 +903,12 @@ def add_variable(self, variable, name, **kwargs):
             else:
                 values = self.process_variable_args(variable, **kwargs)
                 self.variables[variable][name] = values
-
+        elif variable == "adapters":
+            if not self.variable_exists(variable, name):
+                values = self.process_variable_args(variable, **kwargs)
+                self.variables[variable].update(values)
+            else:
+                raise DuplicateConfigVariableError(variable, name)
         else:
             if not self.variable_exists(variable, name):
                 values = self.process_variable_args(variable, **kwargs)
@@ -813,9 +924,19 @@ def add_variable(self, variable, name, **kwargs):
 
     def update_variable(self, variable, name, **kwargs):
         if self.variable_exists(variable, name):
-            values = self.process_variable_args(variable, **kwargs)
-            self.variables[variable][name].update(values)
-
+            # print(f"updating {variable} {name} {kwargs}")
+            values = self.process_variable_args(variable, name=name, **kwargs)
+            if variable == "adapters":
+                self.variables[variable].update(values)
+            elif variable == "adapter_flavors":
+                raise NotImplementedError(
+                    (
+                        "Updating ofadapter flavors is not implemented! "
+                        "please delete the flavor and then add aggain."
+                    )
+                )
+            else:
+                self.variables[variable][name].update(values)
             variable_data = self.variables[variable][name]
         else:
             if variable in ["run_modes", "pucks", "barcode_flavors"]:
@@ -827,7 +948,6 @@ def update_variable(self, variable, name, **kwargs):
         return variable_data
 
     def get_variable(self, variable, name):
-        # print(f"config.get_variable({variable}, {name})")
         if not self.variable_exists(variable, name):
             raise ConfigVariableNotFoundError(variable, name)
         else:
@@ -858,3 +978,500 @@ def get_puck(self, name, return_empty=False):
                 raise
             else:
                 return Puck(name)
+
+    def get_barcode_flavor(self, flavor):
+        return self.get_variable("barcode_flavors", flavor)
+
+    def get_adapter_flavor(
+        self, flavor, add_adapter_sequences=["cut_left", "cut_right"]
+    ):
+        adapter_sequences = self.variables["adapters"]
+        af = self.get_variable("adapter_flavors", flavor)
+        # populate each adapter clip definition in this flavor with the full sequence of the adapter
+        for section, entries in af.items():
+            if section in add_adapter_sequences:
+                for (name, d_adap) in entries.items():
+                    if name == "Q":
+                        continue
+                    d_adap["seq"] = adapter_sequences.get(
+                        name,
+                        f"'{name}' NOT FOUND in 'adapters:' section of config.yaml",
+                    )
+
+        return af
+
+
+def get_run_mode_parser(required=True):
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False,
+        formatter_class=argparse.RawTextHelpFormatter,
+        description="add/update run_mode parent parser",
+        add_help=False,
+    )
+    parser.add_argument(
+        "--name", type=str, help="name of the run_mode to be added", required=True
+    )
+    parser.add_argument(
+        "--parent_run_mode",
+        type=str,
+        help="Name of the parent run_mode. All run_modes will fall back to 'default'",
+    )
+    parser.add_argument(
+        "--umi_cutoff",
+        type=int,
+        nargs="+",
+        help="umi_cutoff for this run_mode."
+        + "the automated analysis will be run with these cutoffs",
+    )
+    parser.add_argument(
+        "--n_beads", type=int, help="number of expected beads for this run_mode"
+    )
+    parser.add_argument(
+        "--clean_dge",
+        required=False,
+        choices=bool_in_str,
+        type=str,
+        help="if True, the DGE will be cleaned of barcodes which overlap with primers",
+    )
+    parser.add_argument(
+        "--detect_tissue",
+        required=False,
+        choices=bool_in_str,
+        type=str,
+        help="By default only beads having at least umi_cutoff UMI counts are analysed "
+        + "during the automated analysis, all other beads are filtered out. If this "
+        + "parameter is set, contiguous islands within umi_cutoff passing beads will "
+        + "also be included in the analysis",
+    )
+    parser.add_argument(
+        "--polyA_adapter_trimming",
+        required=False,
+        choices=bool_in_str,
+        type=str,
+        help="if set, reads will have polyA stretches and adapter sequence overlaps trimmed "
+        + "BEFORE mapping.",
+    )
+    parser.add_argument(
+        "--count_intronic_reads",
+        required=False,
+        choices=bool_in_str,
+        type=str,
+        help="if set, INTRONIC reads will also be countsed (apart from UTR and CDS)",
+    )
+    parser.add_argument(
+        "--count_mm_reads",
+        required=False,
+        choices=bool_in_str,
+        type=str,
+        help="if True, multi-mappers will also be counted. For every "
+        + "multimapper only reads which have one unique read mapped"
+        + "to a CDS or UTR region will be counted",
+    )
+
+    parser.add_argument(
+        "--mesh_data",
+        required=False,
+        choices=bool_in_str,
+        type=str,
+        help="if True, this data will be 'mehsed': a hexagonal structured "
+        + "meshgrid will be created, where each new spot will have diameter"
+        + " of --mesh_spot_diameter_um micron diameter and the spots will "
+        + "be spaced --mesh_spot_distance_um microns apart",
+    )
+    parser.add_argument(
+        "--mesh_type",
+        required=False,
+        choices=["circle", "hexagon"],
+        type=str,
+        help="circle: circles with diameter of --mesh_spot_diameter_um will be placed"
+        + " in a hex grid, where he distance between any two circles will be "
+        + "--mesh_spot_distance_um\n"
+        + "hexagon: a mesh of touching hexagons will be created, with their "
+        + "centers being --mesh_spot_distance_um apart. This will cover all "
+        + "the data without any holes",
+    )
+    parser.add_argument(
+        "--mesh_spot_diameter_um",
+        type=float,
+        required=False,
+        help="diameter of mesh spot, in microns. to create a visium-style "
+        + "mesh, use 55um",
+    )
+    parser.add_argument(
+        "--mesh_spot_distance_um",
+        type=float,
+        required=False,
+        help="distance between mesh spots in um. to create a visium-style "
+        + "mesh use 100um",
+    )
+
+    return parser
+
+
+def get_species_parser(required=True):
+    "a parser that allows to add a reference sequence and annotation, belonging to some species"
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
+    parser.add_argument(
+        "--reference",
+        help="name of the reference (default=genome)",
+        type=str,
+        default="genome",
+    )
+    parser.add_argument("--name", help="name of the species", type=str, required=True)
+    parser.add_argument(
+        "--sequence",
+        help="path to the sequence (.fa) file for the species/reference to be added (e.g. the genome)",
+        type=str,
+        required=required,
+    )
+    parser.add_argument(
+        "--genome",
+        help="[DEPRECATED] path to the genome (.fa) file for the species to be added. --genome=<arg> is a synonym for --reference=genome --sequence=<arg>",
+        type=str,
+        required=False,
+    )
+
+    parser.add_argument(
+        "--annotation",
+        help="path to the genome annotation (.gtf) file for the species to be added",
+        type=str,
+        default="",
+        required=False,
+    )
+    parser.add_argument(
+        "--STAR_index_dir",
+        help="path to STAR index directory",
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "--BT2_index",
+        help="path to BOWTIE2 index",
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "--BT2_flags",
+        help="bt2 mapping arguments for this reference (default=mapping.smk:default_BT2_MAP_FLAGS) ",
+        type=str,
+        default="",
+        required=False,
+    )
+    parser.add_argument(
+        "--STAR_flags",
+        help="STAR mapping arguments for this reference (default=mapping.smk:default_STAR_MAP_FLAGS)",
+        type=str,
+        default="",
+        required=False,
+    )
+
+    return parser
+
+
+def get_barcode_flavor_parser(required=True):
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False, description="add/update barcode_flavor", add_help=False
+    )
+    parser.add_argument(
+        "--name", help="name of the barcode flavor", type=str, required=True
+    )
+    parser.add_argument(
+        "--umi",
+        dest="UMI",
+        help="structure of UMI, using python's list syntax. Example: to set UMI to "
+        + "13-20 NT of Read1, use --UMI r1[12:20]. It is also possible to use the first 8nt of "
+        + "Read2 as UMI: --UMI r2[0:8]",
+        type=str,
+        required=required,
+    )
+    # parser.add_argument(
+    #     "--read1",
+    #     help="can be used to modify or replace 'read1' sequence",
+    #     type=str,
+    # )
+
+    parser.add_argument(
+        "--cell_barcode",
+        dest="cell",
+        help="structure of CELL BARCODE, using python's list syntax. Example: to set"
+        + " the cell_barcode to 1-12 nt of Read1, use --cell_barcode r1[0:12]. It is also possible "
+        + " to reverse the CELL BARCODE, for instance with r1[0:12][::-1] (reversing the first 12nt of"
+        + " Read1, and assigning them as CELL BARCODE).",
+        type=str,
+        required=required,
+    )
+    parser.add_argument(
+        "--seq",
+        help=(
+            "if set, allows to modify the sequence, e.g. "
+            "--seq='r2[4:]' will trim the first 4 bases. "
+            "NOTE: has to be matched with --qual='r2_qual[4:]' !"
+        ),
+        type=str,
+        default=None,
+    )
+    parser.add_argument(
+        "--qual",
+        help=(
+            "if set, allows to modify the base qualities, e.g. "
+            "--qual='r2_qual[4:]' will trim the first 4 quality values"
+            "NOTE: has to be matched with --seq='r2[4:]' !"
+        ),
+        type=str,
+        default=None,
+    )
+    parser.add_argument(
+        "--min_qual_trim",
+        help=(
+            "Especially if you need access to bases 'at the end' of the read, "
+            "that may require trimming bases that bcl2fastq already "
+            "marked as non-existent (i.e. sequencing adapter). "
+            "a good option for that is to clip bases from the 3' end if they "
+            "have very low quality scores (default=disabled)"
+        ),
+        type=int,
+        default=None,
+    )
+
+    return parser
+
+
+def get_adapter_parser(required=True):
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False,
+        add_help=False,
+        description=("add/update adapter sequences"),
+    )
+    parser.add_argument("--name", help="name of the adapter", type=str, required=True)
+    parser.add_argument(
+        "--seq",
+        help="sequence of the adapter",
+        type=str,
+        required=required,
+    )
+    return parser
+
+
+def get_adapter_flavor_parser(required=True):
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False,
+        add_help=False,
+        description="add/update adapter_flavor",
+    )
+    parser.add_argument(
+        "--name",
+        help=(
+            "name of the adapter flavor."
+            "an 'adapter_flavor' describes a recipe for trimming raw reads. "
+            "this can entail clipping sequences from either end (such as TSO "
+            "and/or polyA tails) as well as quality-score based trimming "
+            "of bases on the reads 3' end. \n"
+            "NOTE: adapters need to be first added via \n"
+            "   add_adapter --name <name> --seq <sequence>\n"
+            "before they can be used in an adapter_flavor. To customize the "
+            "parameters for determining a match, key-value pairs can be passed "
+            "to --cut_left and --cut_right, for example:\n\n"
+            "   --cut_left SMART:min_overlap=10:max_errors=0.1 "
+            "   --cut_right Q:min_base_qual=30 polyA:min_overlap=3:max_errors=0.25 "
+        ),
+        type=str,
+        required=True,
+    )
+    parser.add_argument(
+        "--cut_left",
+        help=(
+            "list the adapter sequences that should "
+            "be trimmed from the left side of the read,"
+            "e.g. --cut_left TSO_SMART SMART"
+        ),
+        type=str,
+        nargs="+",
+    )
+    parser.add_argument(
+        "--cut_right",
+        help=(
+            "list the adapter sequences that should "
+            "be trimmed from the right side of the read,"
+            "e.g. --cut_right Q polyG polyA "
+            "NOTE: 'Q' means quality-score trimming"
+        ),
+        type=str,
+        nargs="+",
+    )
+    parser.add_argument(
+        "--paired_end",
+        help=(
+            "if consecutive reads are mates of a paired-end read,"
+            "and trimming leads to a read that is shorter than "
+            "min_read_length, we need to discard either both reads,"
+            "or, instead of discarding, the short read can be replaced"
+            " with a single 'N' [EXPERIMENTAL]"
+        ),
+        type=str,
+        choices=["single_end", "replace_N", "discard_both"],
+    )
+    parser.add_argument(
+        "--min_read_length",
+        help=(
+            "if triming reduces the length of the read below this"
+            " threshold, the read should be discarded (default=18)"
+        ),
+        type=int,
+        default=None,
+    )
+
+    return parser
+
+
+def get_quant_parser(required=True):
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False,
+        add_help=False,
+        description="add/update counting flavors (quant)",
+    )
+    parser.add_argument(
+        "--name",
+        help=(
+            "name of the counting flavor."
+        ),
+        type=str,
+        required=True,
+    )
+    return parser
+
+def get_variable_action_subparsers(parent_parser, variable):
+
+    if variable == "barcode_flavors":
+        variable_singular = variable[:-1]
+        variable_add_update_parser = get_barcode_flavor_parser
+    elif variable == "adapters":
+        variable_singular = variable[:-1]
+        variable_add_update_parser = get_adapter_parser
+    elif variable == "adapter_flavors":
+        variable_singular = variable[:-1]
+        variable_add_update_parser = get_adapter_flavor_parser
+    elif variable == "pucks":
+        variable_singular = variable[:-1]
+        variable_add_update_parser = get_puck_parser
+    elif variable == "run_modes":
+        variable_singular = variable[:-1]
+        variable_add_update_parser = get_run_mode_parser
+    elif variable == "species":
+        variable_singular = variable
+        variable_add_update_parser = get_species_parser
+    elif variable == "quant":
+        variable_singular = variable
+        variable_add_update_parser = get_quant_parser
+
+    else:
+        print(f"suspicious variable {variable}")
+
+    command_help = {
+        "list": f"list {variable} and their settings",
+        "delete": f"delete {variable_singular}",
+        "add": f"add a new {variable_singular}",
+        "update": f"update an existing {variable_singular}",
+    }
+
+    # list command
+    list_parser = parent_parser.add_parser(
+        f"list_{variable}", description=command_help["list"], help=command_help["list"]
+    )
+    list_parser.set_defaults(func=list_variables_cmdline, variable=variable)
+
+    func = add_update_delete_variable_cmdline
+
+    # delete command
+    delete_parser = parent_parser.add_parser(
+        f"delete_{variable_singular}",
+        description=command_help["delete"],
+        help=command_help["delete"],
+    )
+    delete_parser.add_argument(
+        "--name",
+        help=f"name of the {variable_singular} to be deleted",
+        type=str,
+        required=True,
+    )
+    if variable == "species":
+        delete_parser.add_argument(
+            "--reference",
+            help=f"name of the reference to be deleted (genome, rRNA, ...)",
+            type=str,
+            required=True,
+        )
+    delete_parser.set_defaults(func=func, action="delete", variable=variable)
+
+    # add command
+    add_parser = parent_parser.add_parser(
+        f"add_{variable_singular}",
+        parents=[variable_add_update_parser()],
+        description=command_help["add"],
+        help=command_help["add"],
+    )
+    add_parser.set_defaults(func=func, action="add", variable=variable)
+
+    # update command
+    update_parser = parent_parser.add_parser(
+        f"update_{variable_singular}",
+        parents=[variable_add_update_parser(False)],
+        description=command_help["update"],
+        help=command_help["update"],
+    )
+    update_parser.set_defaults(func=func, action="update", variable=variable)
+
+
+def setup_config_parser(parent_parser_subparsers):
+    parser_config = parent_parser_subparsers.add_parser(
+        "config", help="configure spacemake"
+    )
+    parser_config_subparsers = parser_config.add_subparsers(
+        help="config sub-command help"
+    )
+
+    for variable in ConfigFile.main_variables_pl2sg.keys():
+        get_variable_action_subparsers(parser_config_subparsers, variable)
+
+    return parser_config
+
+
+@message_aggregation(logger_name)
+def add_update_delete_variable_cmdline(args):
+    # set the name and delete from dictionary
+    name = args["name"]
+    variable = args["variable"]
+    action = args["action"]
+
+    import yaml
+
+    config = get_global_config()
+    config.assert_main_variable(variable)
+
+    # remove the args from the dict
+    del args["action"]
+    del args["variable"]
+    del args["name"]
+
+    if action == "add":
+        func = config.add_variable
+    elif action == "update":
+        func = config.update_variable
+    elif action == "delete":
+        func = config.delete_variable
+
+    var_variables = func(variable, name, **args)
+    # print and dump config file
+    config.logger.info(yaml.dump(var_variables, sort_keys=False))
+    config.dump()
+
+
+@message_aggregation(logger_name)
+def list_variables_cmdline(args):
+    variable = args["variable"]
+    del args["variable"]
+    import yaml
+
+    config = get_global_config()
+
+    config.logger.info(f"Listing {variable}")
+    config.logger.info(yaml.dump(config.variables[variable]))
diff --git a/spacemake/contrib.py b/spacemake/contrib.py
new file mode 100644
index 00000000..24801265
--- /dev/null
+++ b/spacemake/contrib.py
@@ -0,0 +1,40 @@
+import importlib.metadata
+__version__ = "0.8.0"
+__author__ = ["Nikos Karaiskos", "Tamas Ryszard Sztanka-Toth", "Marvin Jens"]
+__license__ = "GPL"
+__email__ = [
+    "nikolaos.karaiskos@mdc-berlin.de",
+    "tamasryszard.sztanka-toth@mdc-berlin.de",
+    "marvin.jens@charite.de",
+]
+
+author_contributions = """
+SPACEMAKE is built on snakemake scripts originally developed by Nikos Karaiskos
+for the analysis of dropseq data. These gradually evolved into a robust workflow for
+spatial transcriptomics data analysis that was improved and generalized to work
+with different ST technologies by Tamas Ryszard Sztanka-Toth. Marvin Jens contributed
+longread analysis code and support for converting fastq to BAM as a first step.
+Many features of the automated analysis and integration with Novosparc were added by
+Tamas, in close collaboration with Nikos, culminating in the first spacemake
+publication:
+
+    https://doi.org/10.1093/gigascience/giac064
+
+Marvin then added new building blocks to successively replace the java-based 
+dropseq tools with python/pysam based code: cutadapt_bam.py, annotator.py, as well
+as the ability to align raw reads to multiple indices, in close collaboration
+with Nikos & Tamas.
+These changes, together with support for the much larger numbers of spatial barcodes 
+generated by Stereo-seq, seq-scope, et al. will probably lead to a SPACEMAKE2 
+release/paper in the near future.
+"""
+
+roadmap = [
+    ("0.5.5", "universal ST support and utility, novosparc integration. Sztanka-Toth et al. 2022"),
+    ("0.7", "support multiple mapping indices, bulk samples, custom user-defined snakemake rules"),
+    ("1.x", "replace dropseq tools. Own annotator and towards entirely scanpy workflow"),
+    ("1.x", "efficient handling of 1E8+ spatial barcodes (seq-scope etc.)"),
+    ("1.x", "add interactive data exploration support (shiny?)"),
+    ("2.x", "cmdline interface cleanup and remote API support"),
+    ("2.x", "cython magic to speed up parallel BAM processing via shared memory"),
+]
diff --git a/spacemake/data/config/config.yaml b/spacemake/data/config/config.yaml
index a9ef253f..304b888f 100644
--- a/spacemake/data/config/config.yaml
+++ b/spacemake/data/config/config.yaml
@@ -85,15 +85,15 @@ run_modes:
         - 500
     
 
-adapters:
-    smart: 'AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTC'
-    optical_primer: 'GAATCACGATACGTACACCA'
-
 barcode_flavors:
     default:
         cell: "r1[0:12]"
         UMI: "r1[12:20]"
         bam_tags: "CR:{cell},CB:{cell},MI:{UMI},RG:{assigned}"
+    dropseq:
+        cell: "r1[0:12]"
+        UMI: "r1[12:20]"
+        bam_tags: "CR:{cell},CB:{cell},MI:{UMI},RG:{assigned}"
     slide_seq_14bc:
         cell: "r1[0:14]"
         UMI: "r1[14:23]"
@@ -115,6 +115,36 @@ barcode_flavors:
         bam_tags: "CR:{cell},CB:{cell},MI:{UMI},RG:{assigned}"
         cell: "r1[0:20]"
     openst:
-        UMI: r2[0:9]
-        bam_tags: CR:{cell},CB:{cell},MI:{UMI},RG:{assigned}
-        cell: r1[2:27]
+        UMI: "r2[0:9]"
+        bam_tags: "CR:{cell},CB:{cell},MI:{UMI},RG:{assigned}"
+        cell: "r1[2:27]"
+
+adapter_flavors: 
+  dropseq:
+    cut_left:
+    - TSO_SMART:
+        max_error: 0.1
+        min_overlap: 10
+    cut_right:
+    - Q:
+        min_base_quality: 30
+    - polyA:
+        max_error: 0.25
+        min_overlap: 3
+    - polyG:
+        max_error: 0.1
+        min_overlap: 3
+
+adapters:
+adapters:
+    smart: 'AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTC'
+    optical_primer: 'GAATCACGATACGTACACCA'
+    TSO_SMART: AAGCAGTGGTATCAACGCAGAGTGAATGGG
+    SMART: AAGCAGTGGTATCAACGCAGAGTG
+    TSO_10x: AAGCAGTGGTATCAACGCAGAGTACATGGG
+    chromium_bead: CTACACGACGCTCTTCCGATCT
+    dropseq_bead: AAGCAGTGGTATCAACGCAGAGTAC
+    polyA: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+    polyG: GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG
+    nextflex_RA3: TGGAATTCTCGGGTGCCAAGGAACTCCAGTCACTGACCAATCTCGTATGCCGTCTTCTGCTTGAA
+    truseq_RA3: TGGAATTCTCGGGTGCCAAGGAACTCCAGTCACCCGTCCA
diff --git a/spacemake/map_strategy.py b/spacemake/map_strategy.py
new file mode 100644
index 00000000..64665418
--- /dev/null
+++ b/spacemake/map_strategy.py
@@ -0,0 +1,435 @@
+import os
+from collections import defaultdict
+from spacemake.errors import *
+from spacemake.util import dotdict, wc_fill
+from spacemake.snakemake.variables import (
+    complete_data_root,
+    log_dir,
+    star_log_file,
+    star_target_log_file,
+)
+
+map_data = {
+    # used to fetch all info needed to create a BAM file
+    'MAP_RULES_LKUP': {},
+    #   key: path of output BAM file
+    #   value: dotdict with plenty of attributes
+    # these support lookup by (project_id, sample_id)
+    'ANNOTATED_BAMS': defaultdict(set),
+    'REF_NAMES': defaultdict(set),
+    'ALL_BAMS': defaultdict(set),
+    'BAM_IS_NOT_TEMP': set(),
+    'SAMPLE_MAP_STRATEGY': {},
+    # used for symlink name to source mapping
+    'BAM_SYMLINKS': {},
+    'BAM_UNMAPPED_KEEP': set(),
+    # used for automated mapping index generation
+    'INDEX_FASTA_LKUP': {},
+    #   key: bt2_index_file or star_index_file
+    #   value: dotdict with 
+    #       .ref_path: path to genome FASTA
+    #       .ann_path: path to annotation file (GTF) [optional]
+
+    # needed for later stages of SPACEMAKE which require one "star.Log.final.out" file.
+    'STAR_FINAL_LOG_SYMLINKS': {},
+    # We create a symlink from the mapping that was accompanied by the 'final' identifier
+}
+
+#####################################
+#### snakemake string templates #####
+#####################################
+
+ubam_input = "unaligned_bc_tagged.polyA_adapter_trimmed"
+# this must be local and not have .bam appended!
+# basically, if ubam_input were used as {target} in linked_bam it should eval to 
+# "unaligned_bc_tagged.polyA_adapter_trimmed.bam"
+
+# The end-point. Usually annotation-tagged genome alignments
+final_target = "final.polyA_adapter_trimmed"
+
+# patterns for auto-generated BAM file names and symlinks
+linked_bam = complete_data_root + "/{link_name}.bam"
+mapped_bam = complete_data_root + "/{ref_name}.{mapper}.bam"
+unmapped_bam = complete_data_root + "/not_{ref_name}.{mapper}.bam"
+star_mapped_bam = complete_data_root + "/{ref_name}.STAR.bam"
+star_unmapped_bam = complete_data_root + "/not_{ref_name}.STAR.bam"
+bt2_mapped_bam = complete_data_root + "/{ref_name}.bowtie2.bam"
+bt2_unmapped_bam = complete_data_root + "/not_{ref_name}.bowtie2.bam"
+
+bt2_mapped_log = log_dir + "/{ref_name}.bowtie2.log"
+
+# special log file used for rRNA "ribo depletion" stats
+bt2_rRNA_log = log_dir + "/rRNA.bowtie2.bam.log"
+
+# default places for mapping indices, unless specified differently in the config.yaml
+star_index = 'species_data/{species}/{ref_name}/star_index'
+star_index_log = star_index + '.log'
+
+star_index_param = star_index
+star_index_file = star_index + '/SAindex'
+
+bt2_index = 'species_data/{species}/{ref_name}/bt2_index'
+bt2_index_param = bt2_index + '/{ref_name}'
+bt2_index_file = bt2_index_param + '.1.bt2'
+bt2_index_log = bt2_index_param + '.log'
+species_reference_sequence = 'species_data/{species}/{ref_name}/sequence.fa'
+star_idx_service = '{species}.{ref_name}.STAR_index_loaded'
+species_reference_annotation = 'species_data/{species}/{ref_name}/annotation.gtf'
+species_reference_annotation_compiled = 'species_data/{species}/{ref_name}/compiled_annotation'
+species_reference_annotation_compiled_target = 'species_data/{species}/{ref_name}/compiled_annotation/non_overlapping.csv'
+
+#########################
+### Default settings  ###
+#########################
+
+default_BT2_MAP_FLAGS = (
+    " --local"
+    " -L 10 -D 30 -R 30"
+    " --ignore-quals"
+    " --score-min=L,0,1.5" # require 75% of perfect match (2=base match)
+    " -k 10"
+)
+# original rRNA mapping code used --very-fast-local and that was that.
+
+default_STAR_MAP_FLAGS = (
+    # " --genomeLoad NoSharedMemory"
+    " --outSAMprimaryFlag AllBestScore"
+    " --outSAMattributes Standard"
+    " --outSAMunmapped Within"
+    " --outStd BAM_Unsorted"
+    " --outSAMtype BAM Unsorted"
+    " --limitOutSJcollapsed 5000000"
+)
+
+default_counting_flavor_with_annotation = 'default'
+default_counting_flavor_no_annotation = 'custom_index'
+
+#############################################################
+##### Utility functions specific for the mapping module #####
+#############################################################
+
+
+def maybe_temporary(bam):
+    return bam
+    # DOES not currently work, as decision is made at parse time, before
+    # any map-rules have been evaluated. Needs more thought.
+
+    # # print("these are the NOT TEMP bam files", sorted(BAM_IS_NOT_TEMP))
+    # if bam in map_data['BAM_IS_NOT_TEMP']:
+    #     # print(bam, "is not temp!")
+    #     return bam
+    # else:
+    #     # print(bam, "is TEMP!!!1!!11einself")
+    #     return temp(bam)
+
+def get_annotated_bams(wc):
+    # print(wc)
+    return {'annotated_bams' : sorted(map_data['ANNOTATED_BAMS'][(wc.project_id, wc.sample_id)])}
+
+def get_all_mapped_bams(wc):
+    # print(wc)
+    return {'mapped_bams' : sorted(map_data['ALL_BAMS'][(wc.project_id, wc.sample_id)])}
+
+def get_count_flavor_str(wc):
+    cflavors = []
+    for bam in get_all_mapped_bams(wc)['mapped_bams']:
+        mr = map_data['MAP_RULES_LKUP'][bam]
+        cflavors.append(f"{mr.ref_name}@{mr.cflavor}")
+    
+    return ",".join(cflavors)
+
+##########################
+### Core functionality ###
+##########################
+
+def validate_mapstr(mapstr, config={}, species=None):
+    # print(f"validate_mapstr(mapstr={mapstr}, config={config}, species={species})")
+    if species and config:
+        species_d = config.get_variable("species", name=species)
+    else:
+        species_d = {}
+
+    def process(token):
+        """
+        based on the left-hand side name (left) and the rule encoded in token,
+        create one mapping rule and up to one symlink rule.
+        """
+
+        parts = token.split(":")
+        if len(parts) == 2:
+            ref, mapper = parts
+            # old format! We need to swap the order
+            link_name = None
+        elif len(parts) == 3:
+            ref, mapper, link_name = parts
+        else:
+            raise ConfigVariableError(f"map_strategy contains a map-rule with unexpected number of parameters: {parts}")
+
+        if ref in ['bowtie2', 'STAR']:
+            ref, mapper = mapper, ref
+
+        if species_d:
+            if not ref in species_d:
+                raise ConfigVariableNotFoundError(ref, f"reference name '{ref}' is not among the refs registered for {species}: {sorted(species_d.keys())}. Entire map-rule: {mapstr}")
+
+        if '@' in mapper:
+            # we have a counting-flavor directive
+            mapper, cflavor = mapper.split('@')
+            if config:
+                config.assert_variable("quant", cflavor)
+        else:
+            cflavor = None
+        
+        rebuild = f"{ref}:{mapper}"
+        if cflavor:
+            rebuild += f"@{cflavor}"
+        if link_name:
+            rebuild += f":{link_name}"
+
+        return rebuild
+
+    chain_rebuild = []
+    for tokens in mapstr.split("->"):
+        rebuilds = [process(r) for r in tokens.split(",")]
+        chain_rebuild.append(",".join(rebuilds))
+            
+    return "->".join(chain_rebuild)
+
+
+def mapstr_to_targets(mapstr, left="uBAM", final="final"):
+    """
+    Converts a mapping strategy provided as a string into a series of map rules, which translate into 
+    BAM names and their dependencies. Downstream, rule matching is guided by the convention of the 
+    strategy-defined BAM filenames ending in "STAR.bam" or "bowtie2.bam".
+
+    Examples:
+
+        rRNA:bowtie2->genome:STAR:final
+
+    The input to the first mappings is always the uBAM - the CB and UMI tagged, pre-processed, 
+    but unmapped reads.
+    map-rules are composed of two or three paramters, separated by ':'.
+    The first two parameters for the mapping are <reference>:<mapper>. The target BAM will have 
+    the name <reference>.<mapper>.bam.
+    Optionally, a triplet can be used <reference>:<mapper>:<symlink> where the presence of <symlink> 
+    indicates that the BAM file should additionally be made accessible under the name <symlink>.bam, a useful 
+    shorthand or common hook expected by other stages of SPACEMAKE. A special symlink name is "final"
+    which is required by downstream stages of SPACEMAKE. If no "final" is specified, the last map-rule
+    automatically is selected and symlinked as "final".
+    Note, due to the special importance of "final", it may get modified to contain other flags of the run-mode
+    that are presently essential for SPACEMAKE to have in the file name ("final.bam" may in fact be 
+    "final.polyA_adapter_trimmed.bam")
+
+    The example above is going to create
+        
+        (1) rRNA.bowtie2.bam
+        
+            using bowtie2 and the index associated with the "rRNA" reference
+
+
+        (2) genome.STAR.bam 
+        
+            using STAR on the *unmapped* reads from BAM (1)
+
+
+        (3) final.bam
+        
+            a symlink pointing to the actual BAM created in (2).
+
+    Note that one BAM must be designated 'final.bam', or the last BAM file created will be selected as final.
+    (used as input to downstream processing rules for DGE creation, etc.)
+
+    NOTE: Parallel mappings can be implemented by using commata:
+
+        rRNA:bowtie2:rRNA,genome:STAR:final
+
+        This rule differs from the first example because it will align the unmapped reads from the uBAM
+        in parallel to the rRNA reference and to the genome. In this way the same reads can match to both
+        indices.
+
+    NOTE: Gene tagging will be applied automatically if annotation data were provided for the associated 
+    reference index (by using spacemake config add_species --annotation=... )
+    """
+    def process(token, left):
+        """
+        based on the left-hand side name (left) and the rule encoded in token,
+        create one mapping rule and up to one symlink rule.
+        """
+        parts = token.split(":")
+        mr = dotdict()
+        lr = None
+
+        if len(parts) == 2:
+            ref, mapper = parts
+            link_name = None
+        elif len(parts) == 3:
+            ref, mapper, link_name = parts
+            link_name = link_name.replace("final", final)
+        else:
+            raise ValueError(f"map_strategy contains a map-rule with unexpected number of parameters: {parts}")
+        
+        if '@' in mapper:
+            # we have a counting-flavor directive
+            mapper, cflavor = mapper.split('@')
+        else:
+            cflavor = 'auto'
+
+        mr.input_name = left
+        mr.mapper = mapper
+        mr.cflavor = cflavor
+        mr.ref_name = ref
+        mr.out_name = f"{ref}.{mapper}"
+        mr.keep_unmapped = False
+
+        if link_name:
+            lr = dotdict(link_src=mr.out_name, link_name=link_name, ref_name=mr.ref_name)
+
+        return mr, lr
+
+
+    map_rules = []
+    link_rules = []
+
+    chain = mapstr.split("->")
+    final_link = None
+
+    while chain:
+        right = chain.pop(0)
+        if left == right:
+            continue
+
+        last_mr = []
+        for r in right.split(","):
+            mr, lr = process(r, left=left)
+            map_rules.append(mr)
+            last_mr.append(mr)
+            
+            if lr:
+                link_rules.append(lr)
+                # check if we have a "final" mapping
+                if final in lr.link_name:
+                    final_link = lr
+
+        #left = f"not_{mr.out_name}"
+        left = mr.out_name
+
+    for mr in last_mr:
+        mr.keep_unmapped = True
+
+    if not final_link:
+        # we need to manufacture a "final" link_rule by taking the last mapping
+        last = map_rules[-1]
+        link_rules.append(dotdict(link_src=last.out_name, link_name=final, ref_name=last.ref_name))
+
+    # print("generated the following map_rules")
+    # for m in map_rules:
+    #     print(m)
+    return map_rules, link_rules
+
+
+def get_mapped_BAM_output(project_df=None, config=None, default_strategy="genome:STAR:final"):
+    """
+    This function is called from main.smk at least once 
+    to determine which output BAM files need to be generated and 
+    to parse the map_strategy into rules and dependencies.
+    """
+    out_files = []
+
+    for index, row in project_df.df.iterrows():
+        # rules so far are "local" to each sample. Here we create full paths, merging with
+        # project_id/sample_id from the project_df
+
+        map_strategy = getattr(row, "map_strategy", default_strategy)
+        map_data['SAMPLE_MAP_STRATEGY'][index] = map_strategy
+
+        map_rules, link_rules = mapstr_to_targets(map_strategy, left=ubam_input, final=final_target)
+
+        species_d = project_df.config.get_variable("species", name=row.species)
+        is_merged = project_df.get_metadata(
+            "is_merged", project_id=index[0], sample_id=index[1]
+        )
+        if is_merged:
+            continue
+
+        for mr in map_rules:
+            mr.project_id = index[0]
+            mr.sample_id = index[1]
+            mr.species = row.species
+            
+            mr.out_path = wc_fill(mapped_bam, mr)
+            mr.out_unmapped_path = wc_fill(unmapped_bam, mr)
+            if mr.keep_unmapped:
+                map_data['BAM_IS_NOT_TEMP'].add(mr.out_unmapped_path)
+            
+            mr.link_name = mr.input_name
+            mr.input_path = wc_fill(linked_bam, mr)
+
+            mr.ref_path = species_d[mr.ref_name]["sequence"]
+            mr.ann_path = species_d[mr.ref_name].get("annotation", None)
+            map_data['ALL_BAMS'][(index[0], index[1])].add(mr.out_path)
+
+            if mr.cflavor == 'auto':
+                # if we have annotation use the actual default, which works for complex
+                # gene models
+                # if we do NOT have annotation, assume custom_index (rRNA, spikes, CITE-seq tags etc.)
+                mr.cflavor = default_counting_flavor_with_annotation if mr.ann_path else default_counting_flavor_no_annotation
+
+            if mr.ann_path:
+                mr.ann_final = wc_fill(species_reference_annotation, mr)
+                mr.ann_final_compiled = wc_fill(species_reference_annotation_compiled, mr)
+                mr.ann_final_compiled_target = wc_fill(species_reference_annotation_compiled_target, mr)
+
+                # keep track of all annotated BAM files we are going to create
+                # for subsequent counting into DGE matrices/h5ad
+                map_data['ANNOTATED_BAMS'][(index[0], index[1])].add(mr.out_path)
+                map_data['REF_NAMES'][(index[0], index[1])].add(mr.ref_name)
+            else:
+                mr.ann_final = []
+
+            default_STAR_INDEX = wc_fill(star_index, mr)
+            default_BT2_INDEX = wc_fill(bt2_index_param, mr)
+            if mr.mapper == "bowtie2":
+                mr.map_index_param = species_d[mr.ref_name].get("BT2_index", default_BT2_INDEX) # the parameter passed on to the mapper
+                mr.map_index = os.path.dirname(mr.map_index_param) # the index_dir
+                mr.map_index_file = mr.map_index_param + ".1.bt2" # file present if the index is actually there
+                mr.map_flags = species_d[mr.ref_name].get("BT2_flags", default_BT2_MAP_FLAGS)
+
+            elif mr.mapper == "STAR":
+                mr.map_index = species_d[mr.ref_name].get("index_dir", default_STAR_INDEX)
+                mr.map_index_param = mr.map_index
+                mr.map_index_file = mr.map_index + "/SAindex"
+                mr.star_idx_service = star_idx_service.format(**mr)
+                mr.map_flags = species_d[mr.ref_name].get("STAR_flags", default_STAR_MAP_FLAGS)
+
+            map_data['MAP_RULES_LKUP'][mr.out_path] = mr
+            map_data['INDEX_FASTA_LKUP'][mr.map_index_file] = mr
+            #out_files.append(mr.out_path)
+
+        # by convention we keep the last unmapped BAM file in a chain,
+        # the others are temporary
+        map_data['BAM_UNMAPPED_KEEP'].add(mr.out_unmapped_path)
+
+        # process all symlink rules
+        for lr in link_rules:
+            lr.link_path = linked_bam.format(project_id=index[0], sample_id=index[1], link_name=lr.link_name)
+            lr.src_path = linked_bam.format(project_id=index[0], sample_id=index[1], link_name=lr.link_src)
+            map_data['BAM_SYMLINKS'][lr.link_path] = lr.src_path
+
+            if lr.link_name == final_target:
+                final_log_name = star_log_file.format(project_id=index[0], sample_id=index[1])
+                final_log = star_target_log_file.format(ref_name=lr.ref_name, project_id=index[0], sample_id=index[1])
+                # print("STAR_FINAL_LOG_SYMLINKS preparation", final_target, final_log_name, "->", final_log)
+                map_data['STAR_FINAL_LOG_SYMLINKS'][final_log_name] = final_log
+                
+                out_files.append(lr.link_path)
+
+    # for k,v in sorted(map_data['MAP_RULES_LKUP'].items()):
+    #     print(f"map_rules for '{k}'")
+    #     print(v)
+
+    # print("BAM_SYMLINKS")
+    # for k, v in map_data['BAM_SYMLINKS'].items():
+    #     print(f"    output={k} <- source={v}")
+
+    # print("out_files", out_files)
+    return out_files
diff --git a/spacemake/preprocess/__init__.py b/spacemake/preprocess/__init__.py
index 29c73918..b7c2fa72 100644
--- a/spacemake/preprocess/__init__.py
+++ b/spacemake/preprocess/__init__.py
@@ -1,6 +1,6 @@
-from .cmdline import cmdline
-from .dge import calculate_adata_metrics,\
-    calculate_shannon_entropy_scompression, dge_to_sparse_adata,\
-    attach_barcode_file, parse_barcode_file, load_external_dge,\
-    attach_puck_variables, attach_puck
+#from .cmdline import cmdline
+#from .dge import calculate_adata_metrics,\
+    #calculate_shannon_entropy_scompression, dge_to_sparse_adata,\
+    #attach_barcode_file, parse_barcode_file, load_external_dge,\
+    #attach_puck_variables, attach_puck
 
diff --git a/spacemake/preprocess/dge.py b/spacemake/preprocess/dge.py
index ba5dde11..19124e42 100644
--- a/spacemake/preprocess/dge.py
+++ b/spacemake/preprocess/dge.py
@@ -130,6 +130,9 @@ def dge_to_sparse_adata(dge_path, dge_summary_path):
             else:
                  X = hstack([X, gene_sp])
 
+        if X is None:
+            X = coo_matrix((len(barcodes), 0), dtype=np.int32)
+    
         if not has_mt:
             # ensure we have an entry for mitochondrial transcripts even if it's just all zeros
             print(
@@ -161,7 +164,6 @@ def dge_to_sparse_adata(dge_path, dge_summary_path):
 
 
 def load_external_dge(dge_path):
-    import anndata
     import scanpy as sc
 
     from scanpy._utils import check_nonnegative_integers
@@ -216,8 +218,6 @@ def parse_barcode_file(barcode_file):
 
 
 def attach_barcode_file(adata, barcode_file):
-    import pandas as pd
-
     bc = parse_barcode_file(barcode_file)
 
     # new obs has only the indices of the exact barcode matches
@@ -241,6 +241,7 @@ def attach_puck_variables(adata, puck_variables):
 
     x_pos_max, y_pos_max = tuple(adata.obsm["spatial"].max(axis=0))
     x_pos_min, y_pos_min = tuple(adata.obsm["spatial"].min(axis=0))
+    #print(f"PUCK VARS {puck_variables} X MIN {x_pos_min} X MAX {x_pos_max} Y MIN {y_pos_min} Y MAX {y_pos_max}")
 
     width_um = adata.uns["puck_variables"]["width_um"]
     coord_by_um = (x_pos_max - x_pos_min) / width_um
diff --git a/spacemake/project_df.py b/spacemake/project_df.py
index ada05ea9..7ab5a700 100644
--- a/spacemake/project_df.py
+++ b/spacemake/project_df.py
@@ -1,15 +1,15 @@
-import pandas as pd
+# import pandas as pd
 import os
-import yaml
 import math
-import argparse
-import datetime
-import re
+
+# import argparse
+# import datetime
 import logging
 import time
 
 from spacemake.config import Puck
 from spacemake.errors import *
+
 from spacemake.config import ConfigFile
 from spacemake.util import message_aggregation, assert_file, str_to_list
 from spacemake.snakemake.variables import puck_barcode_files_summary
@@ -17,572 +17,6 @@
 
 logger_name = "spacemake.project_df"
 logger = logging.getLogger(logger_name)
-# logging.basicConfig(level=logging.INFO)
-
-
-def get_project_sample_parser(allow_multiple=False, prepend="", help_extra=""):
-    """
-    Return a parser for project_id's and sample_id's
-
-    :param allow_multiple: if true, we allow multiple projects and samples,
-        and the parser will have a `--project_id_list` and a `--sample_id_list`
-        parameter
-    :param prepend: a string that will be prepended before the arguments
-    :param help_extra: extra help message
-    :return: a parser object
-    :rtype: argparse.ArgumentParser
-    """
-    logger.info(f"get_project_sample_parser(prepend={prepend}) called")
-
-    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
-    project_argument = "project_id"
-    sample_argument = "sample_id"
-    required = True
-    nargs = None
-    default = None
-
-    if allow_multiple:
-        project_argument = f"{project_argument}_list"
-        sample_argument = f"{sample_argument}_list"
-        required = False
-        nargs = "*"
-        default = []
-
-    parser.add_argument(
-        f"--{prepend}{project_argument}",
-        type=str,
-        required=required,
-        nargs=nargs,
-        default=default,
-        help=f"{project_argument} {help_extra}",
-    )
-
-    parser.add_argument(
-        f"--{prepend}{sample_argument}",
-        type=str,
-        required=required,
-        nargs=nargs,
-        default=default,
-        help=f"{sample_argument} {help_extra}",
-    )
-
-    return parser
-
-
-def get_add_sample_sheet_parser():
-    """
-    Returns parser for sample sheet addition
-
-    :return: parser
-    :rtype: argparse.ArgumentParser
-    """
-    logger.info("get_add_sample_sheet_parser() called")
-    parser = argparse.ArgumentParser(
-        allow_abbrev=False,
-        description="add a new sample sheet to the samples",
-        add_help=False,
-    )
-
-    parser.add_argument(
-        "--sample_sheet",
-        type=str,
-        help="the path to the Illumina sample sheet",
-        required=True,
-    )
-    parser.add_argument(
-        "--basecalls_dir",
-        type=str,
-        help="path to the basecalls directory",
-        required=True,
-    )
-
-    return parser
-
-
-def get_sample_main_variables_parser(
-    species_required=False,
-    main_variables=["barcode_flavor", "species", "puck", "run_mode", "map_strategy"],
-):
-    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
-    logger.info(
-        f"get_sample_main_variables_parser called with main_variables={main_variables}"
-    )
-
-    if "barcode_flavor" in main_variables:
-        parser.add_argument(
-            "--barcode_flavor", type=str, help="barcode flavor for this sample"
-        )
-
-    if "species" in main_variables:
-        parser.add_argument(
-            "--species",
-            type=str,
-            help="add the name of the species for this sample",
-            required=species_required,
-        )
-
-    if "map_strategy" in main_variables:
-        parser.add_argument(
-            "--map_strategy",
-            type=str,
-            help="string constant definining mapping strategy. Can be multi-stage and use bowtie2 or STAR (see documentation)",
-            required=False,
-            default=None,
-        )
-
-    if "puck" in main_variables:
-        parser.add_argument(
-            "--puck",
-            type=str,
-            help="name of the puck for this sample. if puck_type contains a \n"
-            + "`barcodes` path to a coordinate file, those coordinates\n"
-            + " will be used when processing this sample. if \n"
-            + " not provided, a default puck will be used with \n"
-            + "width_um=3000, spot_diameter_um=10",
-        )
-
-        parser.add_argument(
-            "--puck_barcode_file_id",
-            type=str,
-            help="puck_barcode_file_id of the sample to be added/update",
-            nargs="+",
-        )
-
-    parser.add_argument(
-        "--puck_barcode_file",
-        type=str,
-        nargs="+",
-        help="the path to the file contining (x,y) positions of the barcodes",
-    )
-
-    if "run_mode" in main_variables:
-        parser.add_argument(
-            "--run_mode",
-            type=str,
-            nargs="+",
-            help="run_mode names for this sample.\n"
-            + "the sample will be processed using the provided run_modes.\n"
-            + "for merged samples, if left empty, the run_modes of the \n"
-            + "merged (input) samples will be intersected.\n",
-        )
-
-    return parser
-
-
-def get_sample_extra_info_parser():
-    logger.info("get_sample_extra_info_parser() called")
-    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
-
-    parser.add_argument(
-        "--investigator",
-        type=str,
-        help="add the name of the investigator(s) responsible for this sample",
-    )
-
-    parser.add_argument("--experiment", type=str, help="description of the experiment")
-
-    parser.add_argument(
-        "--sequencing_date",
-        type=datetime.date.fromisoformat,
-        help="sequencing date of the sample. format: YYYY-MM-DD",
-    )
-
-    return parser
-
-
-def get_data_parser(reads_required=False):
-    """
-    Returns a parser which contain extra arguments for a given sample.
-    The returned parser will contain the --R1, --R2, --longreads,
-    --longread-signature, --barcode_flavor, --species, --puck,  --puck_barcode_file_id,
-    --puck_barcode_file, --investigator, --experiment, --sequencing_date,
-    --run_mode arguments.
-
-    :param species_required: if true, the --species argument will be required
-        during parsing.
-    :param reads_required: if true, --R1, --R2, and --longreads arguments will be
-        required during parsing.
-    """
-    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
-    logger.info("get_data_parser() called")
-    parser.add_argument(
-        "--R1",
-        type=str,
-        help=".fastq.gz file path to R1 reads",
-        nargs="+",
-        required=reads_required,
-    )
-
-    parser.add_argument(
-        "--R2",
-        type=str,
-        help=".fastq.gz file path to R2 reads",
-        nargs="+",
-        required=reads_required,
-    )
-
-    parser.add_argument(
-        "--dge",
-        type=str,
-        help="Path to dge matrix. spacemake can also handle already processed"
-        + " digital expression data",
-        required=reads_required,
-    )
-
-    parser.add_argument(
-        "--longreads",
-        type=str,
-        help="fastq(.gz)|fq(.gz)|bam file path to pacbio long reads for library debugging",
-        required=reads_required,
-    )
-
-    parser.add_argument(
-        "--longread-signature",
-        type=str,
-        help="identify the expected longread signature (see longread.yaml)",
-    )
-
-    return parser
-
-
-def get_set_remove_variable_subparsers(
-    parent_parser, variable_name, func, prepend="", allow_multiple=False
-):
-    """get_set_remove_variable_subparsers.
-
-    :param parent_parser:
-    :param variable_name:
-    :param func:
-    :param prepend:
-    :param allow_multiple:
-    """
-    if allow_multiple:
-        nargs = "+"
-    else:
-        nargs = None
-
-    logger.info(f"get_set_remove_variable_subparsers(varname={variable_name}) called")
-
-    def get_action_parser(action):
-        """get_action_parser.
-
-        :param action:
-        """
-        action_parser = parent_parser.add_parser(
-            f"{action}_{variable_name}",
-            parents=[get_project_sample_parser(allow_multiple=True)],
-            description=f"{action} {variable_name} for several projects/samples",
-            help=f"{action} {variable_name} for several projects/samples",
-        )
-        action_parser.add_argument(
-            f"--{variable_name}", type=str, required=True, nargs=nargs
-        )
-        action_parser.set_defaults(func=func, variable=variable_name, action=action)
-
-        return action_parser
-
-    set_parser = get_action_parser("set")
-
-    if allow_multiple:
-        set_parser.add_argument("--keep_old", action="store_true")
-
-        remove_parser = get_action_parser("remove")
-
-
-def get_action_sample_parser(parent_parser, action, func):
-    """get_action_sample_parser.
-
-    :param parent_parser:
-    :param action:
-    :param func:
-    """
-    logger.info(f"get_action_sample_parser(action={action}) called")
-    if action not in ["add", "update", "delete", "merge"]:
-        raise ValueError(f"Invalid action: {action}")
-
-    if action == "merge":
-        parser_name = "merge_samples"
-        msg = "merge samples"
-        parents = [
-            get_project_sample_parser(
-                prepend="merged_", help_extra="of the newly created merged sample"
-            ),
-            get_project_sample_parser(
-                allow_multiple=True, help_extra="of the samples to be merged"
-            ),
-        ]
-    else:
-        parser_name = f"{action}_sample"
-        msg = f"{action} a sample"
-        parents = [get_project_sample_parser()]
-
-    if action == "add":
-        # add arguments for species, run_mode, barcode_flavor and puck
-        parents.append(
-            get_sample_main_variables_parser(
-                species_required=True,
-            )
-        )
-        # add arguments for R1/R1, dge, longread
-        parents.append(get_data_parser())
-        # add arguments for extra sample info
-        parents.append(get_sample_extra_info_parser())
-    elif action == "update":
-        # add main variables parser
-        parents.append(get_sample_main_variables_parser())
-        # add arguments for R1/R1, dge, longread
-        parents.append(get_data_parser())
-        # add possibility to add extra info
-        parents.append(get_sample_extra_info_parser())
-    elif action == "merge":
-        # add main variables parser
-        # when merging, only let the user overwrite puck and run_mode
-        parents.append(
-            get_sample_main_variables_parser(
-                main_variables=["run_mode", "puck"],
-            )
-        )
-
-        # add possibility to add extra info
-        parents.append(get_sample_extra_info_parser())
-
-    sample_parser = parent_parser.add_parser(
-        parser_name, description=msg, help=msg, parents=parents
-    )
-    sample_parser.set_defaults(func=func, action=action)
-
-
-def setup_project_parser(pdf, parent_parser_subparsers):
-    """setup_project_parser.
-
-    :param pdf:
-    :param attach_to:
-    """
-    parser = parent_parser_subparsers.add_parser(
-        "projects",
-        help="manage projects and samples",
-        description="Using one of the subcommands specified below, it is possible to"
-        + " add/update/remove projects and their settings",
-    )
-    subparsers = parser.add_subparsers()
-
-    help_desc = {
-        "add_sample_sheet": "add projects and samples from Illumina sample sheet",
-        "add_samples_from_yaml": "add several samples at once from a .yaml file",
-        "merge_samples": "merge several samples into one. "
-        + "samples need to have the same species",
-        "list": "list several project(s) and sample(s) and their settings",
-    }
-
-    # ADD/UPDATE/DELETE/MERGE sample(s)
-    # get parsers for adding, deleting and updating a sample
-    for action in ["add", "update", "delete"]:
-        get_action_sample_parser(
-            subparsers,
-            action,
-            # attach the function to the parser
-            func=lambda args: add_update_delete_sample_cmdline(pdf, args),
-        )
-
-    # get parser for merging
-    get_action_sample_parser(
-        subparsers, "merge", func=lambda args: merge_samples_cmdline(pdf, args)
-    )
-
-    # LIST PROJECTS
-    # always show these variables
-    always_show = [
-        "puck_barcode_file_id",
-        "species",
-        "investigator",
-        "sequencing_date",
-        "experiment",
-        "run_mode",
-        "barcode_flavor",
-        "map_strategy",
-    ]
-    remaining_options = [
-        x for x in pdf.project_df_default_values.keys() if x not in always_show
-    ]
-    list_projects = subparsers.add_parser(
-        "list",
-        description=help_desc["list"],
-        help=help_desc["list"],
-        parents=[
-            get_project_sample_parser(
-                allow_multiple=True,
-                help_extra="subset the data. If none provided, all will be displayed",
-            )
-        ],
-    )
-    list_projects.add_argument(
-        "--variables",
-        help="which extra variables to display per sample? "
-        + f"{always_show} will always be shown.",
-        choices=remaining_options,
-        default=[],
-        nargs="*",
-    )
-    list_projects.set_defaults(
-        func=lambda args: list_projects_cmdline(pdf, args), always_show=always_show
-    )
-
-    # ADD SAMPLE SHEET
-    sample_add_sample_sheet = subparsers.add_parser(
-        "add_sample_sheet",
-        description=help_desc["add_sample_sheet"],
-        help=help_desc["add_sample_sheet"],
-        parents=[get_add_sample_sheet_parser()],
-    )
-    sample_add_sample_sheet.set_defaults(
-        func=lambda args: add_sample_sheet_cmdline(pdf, args)
-    )
-
-    # ADD SAMPLES FROM YAML
-    sample_add_samples_yaml = subparsers.add_parser(
-        "add_samples_from_yaml",
-        description=help_desc["add_samples_from_yaml"],
-        help=help_desc["add_samples_from_yaml"],
-    )
-    sample_add_samples_yaml.add_argument(
-        "--samples_yaml",
-        type=str,
-        required=True,
-        help="path to the .yaml file containing sample info",
-    )
-    sample_add_samples_yaml.set_defaults(
-        func=lambda args: add_samples_from_yaml_cmdline(pdf, args)
-    )
-
-    # get set/remove parser for each main variable
-    # this will add parser for:
-    # setting/removing run_mode
-    # setting species
-    # setting puck
-    # setting barcode_flavor
-    # NOTE: the ConfigFile.main_variable_sg2type keys determine
-    # which commandline args will be defined for the parser.
-    # this is a little un-intuitive...
-    # TODO: cleaner factory functions for the commandline-parsers
-    for main_var_sg, main_var_type in ConfigFile.main_variable_sg2type.items():
-        allow_multiple = False
-        if isinstance(main_var_type, str) and main_var_type.endswith("_list"):
-            allow_multiple = True
-
-        get_set_remove_variable_subparsers(
-            subparsers,
-            variable_name=main_var_sg,
-            func=lambda args: set_remove_variable_cmdline(pdf, args),
-            allow_multiple=allow_multiple,
-        )
-    return parser
-
-
-@message_aggregation(logger_name)
-def add_update_delete_sample_cmdline(pdf, args):
-    """add_update_delete_sample_cmdline.
-
-    :param pdf:
-    :param args:
-    """
-    action = args["action"]
-
-    # remove the action from args
-    del args["action"]
-
-    if action == "add" or action == "update":
-        func = lambda **kwargs: pdf.add_update_sample(action=action, **kwargs)
-    elif action == "delete":
-        func = pdf.delete_sample
-
-    sample = func(**args)
-    pdf.dump()
-
-
-@message_aggregation(logger_name)
-def add_sample_sheet_cmdline(pdf, args):
-    """add_sample_sheet_cmdline.
-
-    :param pdf:
-    :param args:
-    """
-    pdf.add_sample_sheet(args["sample_sheet"], args["basecalls_dir"])
-
-    pdf.dump()
-
-
-@message_aggregation(logger_name)
-def add_samples_from_yaml_cmdline(pdf, args):
-    """add_samples_from_yaml_cmdline.
-
-    :param pdf:
-    :param args:
-    """
-    pdf.add_samples_from_yaml(args["samples_yaml"])
-
-    pdf.dump()
-
-
-@message_aggregation(logger_name)
-def set_remove_variable_cmdline(pdf, args):
-    """set_remove_variable_cmdline.
-
-    :param pdf:
-    :param args:
-    """
-    variable_name = args["variable"]
-    action = args["action"]
-
-    variable_key = args[variable_name]
-
-    pdf.set_remove_variable(
-        variable_name=variable_name,
-        variable_key=variable_key,
-        action=action,
-        project_id_list=args["project_id_list"],
-        sample_id_list=args["sample_id_list"],
-        keep_old=args.get("keep_old", False),
-    )
-
-    pdf.dump()
-
-
-@message_aggregation(logger_name)
-def merge_samples_cmdline(pdf, args):
-    """merge_samples_cmdline.
-
-    :param pdf:
-    :param args:
-    """
-    pdf.merge_samples(**args)
-
-    pdf.dump()
-
-
-@message_aggregation(logger_name)
-def list_projects_cmdline(pdf, args):
-    """list_projects_cmdline.
-
-    :param pdf:
-    :param args:
-    """
-    projects = args["project_id_list"]
-    samples = args["sample_id_list"]
-    variables = args["always_show"] + args["variables"]
-
-    df = pdf.df
-    logger = logging.getLogger(logger_name)
-
-    if projects != [] or samples != []:
-        df = df.query("project_id in @projects or sample_id in @samples")
-        logger.inof(f"listing projects: {projects} and samples: {samples}")
-    else:
-        logger.info("listing all projects and samples")
-
-    logger.info(f"variables used: {variables}")
-
-    # print the table
-    logger.info(df.loc[:, variables].__str__())
 
 
 class ProjectDF:
@@ -596,6 +30,8 @@ class ProjectDF:
     :type df: pd.DataFrame
     """
 
+    logger = logging.getLogger("spacemake.project_df.ProjectDF")
+
     # default values of the project dataframe columns
     project_df_default_values = {
         "puck_barcode_file_id": ["no_spatial_data"],
@@ -606,6 +42,7 @@ class ProjectDF:
         "basecalls_dir": None,
         "R1": None,
         "R2": None,
+        "reads": None,
         "longreads": None,
         "longread_signature": None,
         "investigator": "unknown",
@@ -619,9 +56,10 @@ class ProjectDF:
         "puck": "default",
         "dge": None,
         "map_strategy": {  # default mapping strategy changes depending on if we have rRNA or not
-            True: "bowtie2:rRNA,STAR:genome:final",  # map in parallel to rRNA and genome (default so far)
-            False: "STAR:genome:final",  # w/o rRNA, map to genome directly
+            True: "rRNA:bowtie2->genome:STAR",  # map in parallel to rRNA and genome (default so far)
+            False: "genome:STAR",  # w/o rRNA, map to genome directly
         },
+        "adapter_flavor": "default",
     }
 
     project_df_dtypes = {
@@ -633,6 +71,7 @@ class ProjectDF:
         "basecalls_dir": "str",
         "R1": "object",
         "R2": "object",
+        "reads": "object",
         "longreads": "str",
         "longread_signature": "str",
         "investigator": "str",
@@ -656,12 +95,13 @@ def __init__(self, file_path, config: ConfigFile = None):
         """
         self.file_path = file_path
         self.config = config
+        import pandas as pd
 
         if os.path.isfile(file_path):
             attempts = 0
-            failed = False
+            df_loaded_flag = False
 
-            while not failed:
+            while not df_loaded_flag:
                 try:
                     self.df = pd.read_csv(
                         file_path,
@@ -669,16 +109,18 @@ def __init__(self, file_path, config: ConfigFile = None):
                         na_values=["None", "none"],
                         dtype=self.project_df_dtypes,
                     )
-                    failed = True
+                    df_loaded_flag = True
                 except pd.errors.EmptyDataError as e:
                     if attempts < 5:
                         # wait 5 seconds before retrying
+                        self.logger.warning(
+                            f"The file '{self.file_path}' seems to be empty. Trying again ({attempts}/5) in 5 seconds"
+                        )
                         time.sleep(5)
                         attempts = attempts + 1
                         continue
                     else:
                         raise e
-                        failed = True
 
             if self.df.empty:
                 self.create_empty_df()
@@ -688,6 +130,8 @@ def __init__(self, file_path, config: ConfigFile = None):
         else:
             self.create_empty_df()
 
+        self.assert_valid()
+
         self.logger = logging.getLogger(logger_name)
 
     def assert_valid(self):
@@ -710,6 +154,10 @@ def assert_valid(self):
         elif self.df.empty:
             logger.warn("The 'project_df' in the ProjectDF object is empty")
 
+        _unique_samples = self._check_unique_samples()
+        if _unique_samples is not None:
+            raise SampleAlreadyExistsError(_unique_samples)
+
         for index, row in self.df.iterrows():
             # check that sample sheet file exists
             if (row['sample_sheet'] is not None) and (not os.path.exists(row['sample_sheet'])):
@@ -731,10 +179,10 @@ def assert_valid(self):
                 if type(row[n_col]) is list:
                     for _it_row in row[n_col]:
                         if not os.path.exists(_it_row):
-                            raise SystemExit(FileNotFoundError(f"At {index}, the {n_col} file does not exist"))
+                            raise SystemExit(FileNotFoundError(f"At {index}, the {n_col} file does not exist: '{os.path.abspath(_it_row)}'"))
                 elif type(row[n_col]) is str and row[n_col] != '':
                     if not os.path.exists(row[n_col]):
-                        raise SystemExit(FileNotFoundError(f"At {index}, the {n_col} file does not exist"))
+                        raise SystemExit(FileNotFoundError(f"At {index}, the {n_col} file does not exist: '{os.path.abspath(_it_row)}'"))
                     
             # check that pucks are specified only if puck is spatial (or puck_collection is enabled)
             _valid_puck_coordinate = True
@@ -764,12 +212,21 @@ def assert_valid(self):
                     
             if not _valid_puck_coordinate:
                 _puck_vars = self.get_puck_variables(project_id = index[0], sample_id = index[1])
-                if _puck_vars['coordinate_system'] != '':
+                if _puck_vars.get('coordinate_system', '') != '':
                     raise SystemExit(SpacemakeError(f"At {index}, the selected puck '{row['puck']}' " + \
                                                     "contains a coordinate_system " + \
                                                     "but no 'puck_barcode_files' are specified"))
+                
+    def _check_unique_samples(self):
+        for index, _ in self.df.iterrows():
+            if len(self.df[self.df.index.isin([index])]) > 1:
+                return index
+        
+        return None
 
     def create_empty_df(self):
+        import pandas as pd
+
         index = pd.MultiIndex(
             names=["project_id", "sample_id"], levels=[[], []], codes=[[], []]
         )
@@ -851,6 +308,75 @@ def find_dir(name, path):
         else:
             return self.project_df_default_values["puck_barcode_file"]
 
+    def consolidate_pucks_merged_samples(self):
+        for index, row in self.df.iterrows():
+            project_id, sample_id = index
+            puck_ids = row['puck_barcode_file_id']
+            
+            if (not row['is_merged']) or (not self.is_spatial(project_id, sample_id, puck_ids)):
+                continue
+
+            if len(puck_ids) >= 1:
+                puck_ids = puck_ids[0]
+            elif len(puck_ids):
+                puck_ids = self.project_df_default_values['puck_barcode_file_id']
+
+            merged_from = row['merged_from']
+            puck_id_file = set()
+
+            for sample_tuple in merged_from:
+                pid = self.df.loc[sample_tuple]['puck_barcode_file_id']
+                pbf = self.df.loc[sample_tuple]['puck_barcode_file']
+                _tuple = [(id, bf) for id, bf in zip(pid, pbf)]
+                
+                puck_id_file.update([ tuple(t) for t in _tuple ])
+
+            pid, pbf = list(zip(*list(puck_id_file)))
+            self.df.loc[index, 'puck_barcode_file_id'] = list(pid)
+            self.df.loc[index, 'puck_barcode_file'] = list(pbf)
+
+
+    def update_project_df_barcode_matches(self, prealigned=False):
+        from spacemake.snakemake.variables import puck_count_barcode_matches_summary, puck_count_prealigned_barcode_matches_summary
+        import pandas as pd
+
+        if prealigned:
+            _bc_file = puck_count_prealigned_barcode_matches_summary
+        else:
+            _bc_file = puck_count_barcode_matches_summary
+
+        for index, row in self.df.iterrows():
+            project_id, sample_id = index
+
+            puck_ids = row['puck_barcode_file_id']
+            if len(puck_ids) >= 1:
+                puck_ids = puck_ids[0]
+            elif len(puck_ids):
+                puck_ids = self.project_df_default_values['puck_barcode_file_id']
+
+            if (row['is_merged'] and prealigned) or (not self.is_spatial(project_id, sample_id, puck_ids)):
+                continue
+
+            # check if barcodes have been filtered
+            _f_barcodes_df = _bc_file.format(project_id=project_id,
+                    sample_id=sample_id)
+
+            # project_df is only updated if a prealignment barcode matching file is found
+            if os.path.exists(_f_barcodes_df):
+                barcodes_df = pd.read_csv(_f_barcodes_df)
+
+                if 'pass_threshold' not in barcodes_df.columns:
+                    continue
+
+                above_threshold_mask = barcodes_df['pass_threshold'] == 1
+
+                _puck_barcode_files = barcodes_df[above_threshold_mask]['puck_barcode_file'].values.tolist()
+                _puck_barcode_files_id = barcodes_df[above_threshold_mask]['puck_barcode_file_id'].values.tolist()
+
+                self.df.at[index, 'puck_barcode_file'] = _puck_barcode_files
+                self.df.at[index, 'puck_barcode_file_id'] = _puck_barcode_files_id
+
+
     def get_sample_info(self, project_id: str, sample_id: str) -> Dict:
         """get_sample_info.
 
@@ -883,6 +409,7 @@ def is_external(self, project_id: str, sample_id: str) -> bool:
             [
                 "R1",
                 "R2",
+                "reads",
                 "basecalls_dir",
                 "sample_sheet",
                 "longreads",
@@ -899,9 +426,10 @@ def is_external(self, project_id: str, sample_id: str) -> bool:
             + f"is_merged={bool(data.is_merged)}"
         )
         if (
-            (data.R1 and data.R2)
+            data.R2  # R1 is optional (bulk samples)
             or (data.basecalls_dir and data.sample_sheet)
             or (data.longreads)
+            or (data.reads)
             and not data.dge
             or data.is_merged
         ):
@@ -930,6 +458,7 @@ def has_dge(self, project_id: str, sample_id: str) -> bool:
             [
                 "R1",
                 "R2",
+                "reads",
                 "basecalls_dir",
                 "sample_sheet",
                 "longreads",
@@ -940,9 +469,10 @@ def has_dge(self, project_id: str, sample_id: str) -> bool:
 
         if (
             data.is_merged
-            or (data.R1 and data.R2)
+            or data.R2
             or (data.sample_sheet and data.basecalls_dir)
             or data.dge
+            or data.reads
         ):
             return True
         elif data.longreads:
@@ -982,14 +512,16 @@ def is_spatial(
     def get_default_map_strategy_for_species(self, species):
         have_rRNA = "rRNA" in self.config.variables["species"][species]
         map_strategy = self.project_df_default_values["map_strategy"][have_rRNA]
-        print(
-            f"getting default for '{species}' have_rRNA={have_rRNA} -> {map_strategy}"
-        )
+        # print(
+        #     f"getting default for '{species}' have_rRNA={have_rRNA} -> {map_strategy}"
+        # )
         return map_strategy
 
     def fix(self):
         import numpy as np
+        import pandas as pd
 
+        modified = False
         # convert types
         self.df = self.df.where(pd.notnull(self.df), None)
         self.df = self.df.replace({np.nan: None})
@@ -1015,18 +547,48 @@ def fix(self):
 
         project_list = []
         # required if upgrading from pre-longread tree
+        if not "reads" in self.df.columns:
+            modified = True
+            self.df["reads"] = None
+
         if not "longreads" in self.df.columns:
+            modified = True
             self.df["longreads"] = None
 
         if not "longread_signature" in self.df.columns:
+            modified = True
             self.df["longread_signature"] = None
 
         # required if upgrading from pre-bowtie2/map-strategy tree
         if not "map_strategy" in self.df.columns:
+            modified = True
             self.df["map_strategy"] = self.df["species"].apply(
                 self.get_default_map_strategy_for_species
             )
 
+        # validate and correct map_strategies
+        from spacemake.map_strategy import validate_mapstr
+        corrected_map_strategies = []
+        for row in self.df.itertuples():
+            corrected_map_strategies.append(
+                validate_mapstr(row.map_strategy, config=self.config, species=row.species)
+            )
+        self.df['map_strategy'] = corrected_map_strategies
+
+        if not "adapter_flavor" in self.df.columns:
+            self.df["adapter_flavor"] = self.project_df_default_values["adapter_flavor"]
+            # print("added adapter-flavor!")
+            modified = True
+
+        if modified:
+            self.logger.warning(
+                f".fix() reported changes! Saving migrated project_df.csv to '{self.file_path}'"
+            )
+            # self.logger.warning(self.df)
+            # self.logger.warning(self.df.columns)
+            # self.logger.warning(self.df["adapter_flavor"])
+            self.dump()
+
         # per row updates
         # first create a series of a
         for ix, row in self.df.iterrows():
@@ -1118,11 +680,14 @@ def get_matching_puck_barcode_file_ids(
         project_id: str,
         sample_id: str,
     ):
+        import pandas as pd
+
         summary_file = puck_barcode_files_summary.format(
             project_id=project_id, sample_id=sample_id
         )
 
         if not os.path.isfile(summary_file):
+            # print(f"looking for summary file: '{summary_file}'")
             return self.project_df_default_values["puck_barcode_file_id"]
 
         df = pd.read_csv(summary_file)
@@ -1152,6 +717,7 @@ def get_puck_barcode_file_metrics(
 
         if not os.path.isfile(summary_file):
             return None
+        import pandas as pd
 
         df = pd.read_csv(summary_file)
 
@@ -1246,6 +812,7 @@ def get_metadata(self, field, project_id=None, sample_id=None, **kwargs):
 
     def dump(self):
         """dump."""
+        self.logger.debug(f"writing project_df.csv to '{self.file_path}")
         self.df.to_csv(self.file_path)
 
     def add_sample_sheet(self, sample_sheet_path, basecalls_dir):
@@ -1254,6 +821,8 @@ def add_sample_sheet(self, sample_sheet_path, basecalls_dir):
         :param sample_sheet_path:
         :param basecalls_dir:
         """
+        import pandas as pd
+
         with open(sample_sheet_path) as sample_sheet:
             ix = 0
             investigator = None
@@ -1352,6 +921,7 @@ def add_update_sample(
         sample_id=None,
         R1=None,
         R2=None,
+        reads=None,
         dge=None,
         longreads=None,
         longread_signature=None,
@@ -1371,6 +941,7 @@ def add_update_sample(
         :param sample_id:
         :param R1:
         :param R2:
+        :param reads:
         :param dge:
         :param longreads:
         :param longread_signature:
@@ -1381,6 +952,8 @@ def add_update_sample(
         :param map_strategy:
         :param kwargs:
         """
+        import pandas as pd
+
         ix = (project_id, sample_id)
         sample_exists = self.sample_exists(*ix)
 
@@ -1407,8 +980,18 @@ def add_update_sample(
         # If longreads not provided, we try with basecalls_dir and sample_sheet
         #   (only used by add_sample_sheet command)
         # If those area also not provided, we try to add a simple dge
-        if action == "add" and (R1 is None or R2 is None) and not is_merged:
-            self.logger.info("R1 or R2 not provided, trying longreads")
+
+        if R1 == ["None"]:
+            R1 = None
+
+        if R2 == ["None"]:
+            R2 = None
+        
+        if reads == "None":
+            reads = None
+        
+        if action == "add" and (R2 is None) and (reads is None) and not is_merged:
+            self.logger.info("R2 not provided, trying longreads")
 
             if not longreads:
                 self.logger.info(
@@ -1420,7 +1003,7 @@ def add_update_sample(
                     )
                     if not dge:
                         raise SpacemakeError(
-                            "Neither R1 & R2, longreads, basecalls_dir & "
+                            "Neither R1,R2, longreads, basecalls_dir & "
                             + "sample_sheet, nor dge were provided.\n"
                             + "Some reads/data has to be provided"
                         )
@@ -1431,8 +1014,13 @@ def add_update_sample(
                     )
 
         # assert files first
+        # if R1 is not None:
         assert_file(R1, default_value=None, extension=".fastq.gz")
+
+        # if R2 is not None:
         assert_file(R2, default_value=None, extension=".fastq.gz")
+
+        assert_file(reads, default_value=None, extension=[".txt", ".csv", ".tsv"])
         assert_file(longreads, default_value=None, extension="all")
         assert_file(
             dge,
@@ -1470,6 +1058,7 @@ def add_update_sample(
         config_variables_to_check = {
             "pucks": "puck",
             "barcode_flavors": "barcode_flavor",
+            "adapter_flavors": "adapter_flavor",
             "species": "species",
         }
 
@@ -1482,6 +1071,7 @@ def add_update_sample(
         # first populate kwargs
         kwargs["R1"] = R1
         kwargs["R2"] = R2
+        kwargs["reads"] = reads
         kwargs["dge"] = dge
         kwargs["longreads"] = longreads
         kwargs["longread_signature"] = longread_signature
@@ -1503,7 +1093,17 @@ def add_update_sample(
                 )
             )
 
-        kwargs["map_strategy"] = map_strategy
+        if action == "add":
+            _i_species = kwargs["species"]
+        elif action == "update":
+            _i_species = self.df.loc[ix]['species']
+
+        # validate and correct map_strategies
+        from spacemake.map_strategy import validate_mapstr
+        kwargs['map_strategy'] = validate_mapstr(map_strategy, config=self.config, species=_i_species)
+
+        # TODO: remove
+        # kwargs["map_strategy"] = map_strategy
 
         # populate puck_barcode_file
         if puck_barcode_file is not None:
@@ -1617,6 +1217,8 @@ def add_samples_from_yaml(self, projects_yaml_file):
 
         :param projects_yaml_file:
         """
+        import yaml
+
         config = yaml.load(open(projects_yaml_file), Loader=yaml.FullLoader)
         demux_projects = config.get("projects", None)
 
@@ -1788,6 +1390,9 @@ def merge_samples(
 
         consistent_variables = list(self.config.main_variables_sg2pl.keys())
         consistent_variables.remove("run_mode")
+        consistent_variables.remove("adapter")
+        consistent_variables.remove("quant")
+
         
         # append variable "map_strategy" manually.
         # TODO: consider moving "map_strategy" into the main_variables_sg2pl, needs additional parser etc.
@@ -1885,3 +1490,17 @@ def merge_samples(
         )
 
         return (sample_added, ix)
+
+
+__global_ProjectDF = None
+
+
+def get_global_ProjectDF(root="."):
+    global __global_ProjectDF
+    if __global_ProjectDF is None:
+        from spacemake.config import get_global_config
+        __global_ProjectDF = ProjectDF(
+            f"{root}/project_df.csv", config=get_global_config()
+        )
+
+    return __global_ProjectDF
diff --git a/spacemake/smk.py b/spacemake/smk.py
index fd41a0dd..766a3180 100644
--- a/spacemake/smk.py
+++ b/spacemake/smk.py
@@ -1,29 +1,9 @@
-import re
-import sys
-import os
-import snakemake
-import argparse
-import yaml
 import logging
-import scanpy as sc
-import pandas as pd
-import anndata
-
-from shutil import copyfile
-from spacemake.project_df import ProjectDF, get_project_sample_parser
-from spacemake.config import ConfigFile
-from spacemake.util import message_aggregation
-from spacemake.errors import SpacemakeError
-from spacemake.preprocess import attach_puck_variables
-from spacemake.spatial.novosparc_integration import get_spacemake_parser as \
-    novosparc_spacemake_parser
-
-config_path = "config.yaml"
-project_df = "project_df.csv"
 
 logger_name = "spacemake.main"
 logger = logging.getLogger(logger_name)
 
+
 class Spacemake:
     """Spacemake.
 
@@ -33,20 +13,20 @@ class Spacemake:
 
     def __init__(self, root):
         """__init__ constructor function of the Spacemake class
-        
+
         :param root: Path to the spacemake root directory.
         :type root: str
         """
+        from spacemake.config import get_global_config
+        from spacemake.project_df import get_global_ProjectDF
+
         self.root = root
-        self.config = ConfigFile.from_yaml(f'{root}/config.yaml')
-        self.project_df = ProjectDF(f'{root}/project_df.csv', config=self.config)
-    
-    def load_processed_adata(self,
-        project_id,
-        sample_id,
-        run_mode_name,
-        umi_cutoff 
-    ) -> anndata.AnnData:
+        self.config = get_global_config(root)
+        self.project_df = get_global_ProjectDF(root)
+
+    def load_processed_adata(
+        self, project_id, sample_id, run_mode_name, umi_cutoff
+    ): #-> anndata.AnnData:
         """Load spacemake processed data.
 
         :param project_id: project_id of the data to be loaded.
@@ -57,7 +37,7 @@ def load_processed_adata(self,
             Each sample can have several run_modes during sample addition,
             here only one option needs to be provided.
         :type run_mode_name: str
-        :param umi_cutoff: the umi_cutoff of the data to be loaded. Each 
+        :param umi_cutoff: the umi_cutoff of the data to be loaded. Each
             run_mode can have several umi_cutoffs provided during configuration
             here only one option needs to be provided.
         :type umi_cutoff: int
@@ -65,39 +45,41 @@ def load_processed_adata(self,
             the results of the analysis.
         :rtype: anndata.AnnData
         """
+        import scanpy as sc
+        # import anndata
+
+
         self.project_df.assert_run_mode(project_id, sample_id, run_mode_name)
         run_mode = self.config.get_run_mode(run_mode_name)
 
-        if not int(umi_cutoff) in [int(uc)
-            for uc in run_mode.variables['umi_cutoff']]:
-            raise SpacemakeError(f'run_mode={run_mode} has no ' + 
-                f'umi_cutoff={umi_cutoff}')
+        if not int(umi_cutoff) in [int(uc) for uc in run_mode.variables["umi_cutoff"]]:
+            raise SpacemakeError(
+                f"run_mode={run_mode} has no " + f"umi_cutoff={umi_cutoff}"
+            )
 
         adata_raw = self.load_raw_spatial_adata(
-            project_id = project_id,
-            sample_id = sample_id,
-            run_mode_name = run_mode_name)
-
-        adata = sc.read(f'{self.root}/projects/{project_id}/processed_data/{sample_id}/'+
-                f'illumina/complete_data/automated_analysis/{run_mode_name}/' +
-                f'umi_cutoff_{umi_cutoff}/results.h5ad')
-        
-        if 'run_mode_variables' not in adata.uns.keys():
-            adata.uns['run_mode_variables'] = run_mode.variables
-        if 'puck_variables' not in adata.uns.keys():
-            adata.uns['puck_variables'] = adata_raw.uns['puck_variables']
-        
+            project_id=project_id, sample_id=sample_id, run_mode_name=run_mode_name
+        )
+
+        adata = sc.read(
+            f"{self.root}/projects/{project_id}/processed_data/{sample_id}/"
+            + f"illumina/complete_data/automated_analysis/{run_mode_name}/"
+            + f"umi_cutoff_{umi_cutoff}/results.h5ad"
+        )
+
+        if "run_mode_variables" not in adata.uns.keys():
+            adata.uns["run_mode_variables"] = run_mode.variables
+        if "puck_variables" not in adata.uns.keys():
+            adata.uns["puck_variables"] = adata_raw.uns["puck_variables"]
+
         return adata
-    
+
     def load_raw_spatial_adata(
-        self,
-        project_id,
-        sample_id,
-        run_mode_name
-    ) -> anndata.AnnData:
+        self, project_id, sample_id, run_mode_name
+    ): #-> anndata.AnnData:
         """Load raw, spacemake processed data.
 
-        This function will load the raw countr matrix, created by spacemake. 
+        This function will load the raw countr matrix, created by spacemake.
 
         :param project_id: project_id of the raw data to be loaded.
         :type project_id: str
@@ -111,6 +93,8 @@ def load_raw_spatial_adata(
             raw expression data, and all cells or spatial units in the dataset.
         :rtype: anndata.AnnData
         """
+        import scanpy as sc
+
         self.project_df.assert_run_mode(project_id, sample_id, run_mode_name)
         run_mode = self.config.get_run_mode(run_mode_name)
 
@@ -133,234 +117,27 @@ def load_raw_spatial_adata(
         if run_mode.variables["clean_dge"]:
             dge_cleaned = ".cleaned"
 
-        adata = sc.read(f'{self.root}/projects/{project_id}/processed_data/{sample_id}/'+
-                f'illumina/complete_data/dge/dge{dge_type}{dge_cleaned}'+
-                f'{polyA_adapter_trimmed}{mm_included}.spatial_beads.h5ad')
-        
-        if 'puck_variables' not in adata.uns.keys():
-            adata = attach_puck_variables(adata,
-                puck_variables = self.project_df.get_puck_variables(
-                    project_id = project_id,
-                    sample_id = sample_id
-                )
-            )
-
-        if 'run_mode_variables' not in adata.uns.keys():
-            adata.uns['run_mode_variables'] = run_mode.variables
-
-        return adata
-
-def get_run_parser():
-    """get_run_parser."""
-    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
-
-    parser.add_argument(
-        "--cores", default=1, type=int, help="number of cores to be used in total"
-    )
-    parser.add_argument(
-        "--dryrun",
-        "-n",
-        action="store_true",
-        help="invokes a dry snakemake run, printing only commands",
-    )
-    parser.add_argument(
-        "--rerun-incomplete",
-        "--ri",
-        action="store_true",
-        help="forces snakemake to rerun incompletely generated files",
-    )
-    parser.add_argument(
-        "--keep-going",
-        action="store_true",
-        help="if a job fails, keep executing independent jobs",
-    )
-    parser.add_argument(
-        "--printshellcmds",
-        "-p",
-        action="store_true",
-        help="print shell commands for each rule, if exist",
-    )
-    parser.add_argument(
-        "--touch",
-        "-t",
-        action="store_true",
-        help="rather than running the rules, just touch each file",
-    )
-    parser.add_argument(
-        "--with_fastqc",
-        "-wfqc",
-        action="store_true",
-        help="Run also fastqc as part of the spacemake run",
-    )
-
-    return parser
-
-def setup_init_parser(parent_parser_subparsers):
-    """setup_init_parser.
-
-    :param parent_parser_subparsers:
-    """
-    parser_init = parent_parser_subparsers.add_parser(
-        "init",
-        help="initialise spacemake: create config files, download genomes and annotations",
-    )
-    parser_init.add_argument(
-        "--root_dir",
-        default="",
-        help="where to output the results of the spacemake run. defaults to .",
-    )
-    parser_init.add_argument(
-        "--temp_dir",
-        default="/tmp",
-        help="temporary directory used when running spacemake. defaults to /tmp",
-    )
-    parser_init.add_argument(
-        "--download_species",
-        default=False,
-        help="if set, upon initialisation, spacemake will download the mouse and human genome and index",
-        action="store_true",
-    )
-    parser_init.add_argument(
-        "--dropseq_tools",
-        help="absolute path to dropseq_tools directory",
-        required=True,
-    )
-    parser_init.set_defaults(func=spacemake_init)
-
-    return parser_init
-
-def setup_run_parser(pdf, parent_parser_subparsers):
-    """setup_run_parser.
-
-    :param pdf:
-    :param parent_parser_subparsers:
-    """
-    parser_run = parent_parser_subparsers.add_parser(
-        "run", help="run spacemake", parents=[get_run_parser()]
-    )
-    parser_run.set_defaults(func=lambda args: spacemake_run(pdf, args))
-
-    # create subparsers
-    parser_run_subparsers = parser_run.add_subparsers()
-
-    downsampling_parser = parser_run_subparsers.add_parser(
-        "downsample",
-        help="run downsampling analysis for a list of projects/samples",
-        parents=[get_project_sample_parser(allow_multiple=True), get_run_parser()],
-    )
-    downsampling_parser.set_defaults(downsample=True,
-        func=lambda args: spacemake_run(pdf, args))
-
-    #parser_novosparc = novosparc_spacemake_parser(parser_run_subparsers)
-    #parser_novosparc.set_defaults(novosparc_reconstruct=True,
-    #    func=lambda args: spacemake_run(pdf, args))
-    
-    return parser_run
-
-@message_aggregation(logger_name)
-def spacemake_init(args):
-    """spacemake_init.
-
-    :param args:
-    """
-    if os.path.isfile(config_path):
-        msg = "spacemake has already been initiated in this directory.\n"
-        msg += "use other commands to run and analyse samples."
-        logger.info(msg)
-        return 0
-    initial_config = os.path.join(os.path.dirname(__file__), "data/config/config.yaml")
-
-    # initialise config file
-    cf = ConfigFile.from_yaml(initial_config)
-    # update the file path
-    cf.set_file_path(config_path)
-
-    # add variables from args
-    cf.variables["root_dir"] = args["root_dir"]
-    cf.variables["temp_dir"] = args["temp_dir"]
-
-    cf.variables['external_bin'] = {
-        'dropseq_tools' : args['dropseq_tools']
-    }
-
-    cf.variables['microscopy_out'] = args.get('microscopy_out', '')
-
-    if args["download_species"]:
-        species_data_config_file = os.path.join(
-            os.path.dirname(__file__), "data/config/species_data_url.yaml"
-        )
-        species_data_config = yaml.load(
-            open(species_data_config_file), Loader=yaml.FullLoader
+        adata = sc.read(
+            f"{self.root}/projects/{project_id}/processed_data/{sample_id}/"
+            + f"illumina/complete_data/dge/dge{dge_type}{dge_cleaned}"
+            + f"{polyA_adapter_trimmed}{mm_included}.spatial_beads.h5ad"
         )
 
-        # add keys as species to config
-        species = list(species_data_config.keys())
-        snakemake_config = {"root_dir": ""}
-        snakemake_config["species"] = species
-
-        # define the pattern
-        snakemake_config[
-            "annotation_file_pattern"
-        ] = "species_data/{species}/{species}_{data_type}.gtf"
-        snakemake_config[
-            "genome_file_pattern"
-        ] = "species_data/{species}/{species}_{data_type}.fa"
-
-        # the to be saved file paths
-        species_info = {}
-        species_files_exist = []
-
-        for sp in species:
-            if not sp in species_info.keys():
-                species_info[sp] = {}
-            for data_type in ["genome", "annotation"]:
-                # save download url to be passed to snakemake
-                snakemake_config[sp + "_" + data_type + "_url"] = species_data_config[
-                    sp
-                ][data_type]
-                # save file path of the result
-                species_file = snakemake_config[data_type + "_file_pattern"].format(
-                    species=sp, data_type=data_type
-                )
-                species_info[sp][data_type] = species_file
-
-                # add bool if file exists
-                species_files_exist.append(os.path.isfile(species_file))
-
-        # if any of the files are missing
-        if not all(species_files_exist):
-            # get the snakefile for downloading the .gtf and .fa files
-            snakefile = os.path.join(
-                os.path.dirname(__file__), "snakemake/species_init.smk"
-            )
-            # run snakemake: download species data and place them in the right place
-            snakemake.snakemake(snakefile, cores=1, config=snakemake_config)
+        if "puck_variables" not in adata.uns.keys():
+            from spacemake.preprocess import attach_puck_variables
 
-        for key, value in species_info.items():
-            cf.add_variable(
-                "species", key, reference="genome", sequence=value["genome"], annotation=value["annotation"]
+            adata = attach_puck_variables(
+                adata,
+                puck_variables=self.project_df.get_puck_variables(
+                    project_id=project_id, sample_id=sample_id
+                ),
             )
 
-    # copy visium_puck_barcode_file
-    dest_visium_path = "puck_data/visium_barcode_positions.csv"
-    logger.info(f"Moving visium barcodes to {dest_visium_path}")
-    os.makedirs(os.path.dirname(dest_visium_path), exist_ok=True)
-    copyfile(
-        os.path.join(os.path.dirname(__file__), "data/visium_barcode_positions.csv"),
-        dest_visium_path,
-    )
-
-    # copy openst_coordinate_system
-    dest_puck_collection_path = "puck_data/openst_coordinate_system.csv"
-    logger.info(f"Moving puck collection coordinate system to {dest_puck_collection_path}")
-    os.makedirs(os.path.dirname(dest_puck_collection_path), exist_ok=True)
-    copyfile(
-        os.path.join(os.path.dirname(__file__), "data/puck_collection/openst_coordinate_system.csv"),
-        dest_puck_collection_path,
-    )
-
-    # save
-    cf.dump()
+        if "run_mode_variables" not in adata.uns.keys():
+            adata.uns["run_mode_variables"] = run_mode.variables
+
+        return adata
+
 
 def get_novosparc_variables(pdf, args):
     """get_novosparc_variables.
@@ -369,9 +146,9 @@ def get_novosparc_variables(pdf, args):
     :param args:
     """
     # assert that sample exists
-    pdf.assert_sample(args['project_id'], args['sample_id'])
+    pdf.assert_sample(args["project_id"], args["sample_id"])
 
-    def populate_variables_from_args(pdf, args, arg_prefix=''):
+    def populate_variables_from_args(pdf, args, arg_prefix=""):
         """populate_variables_from_args.
 
         :param pdf:
@@ -380,345 +157,78 @@ def populate_variables_from_args(pdf, args, arg_prefix=''):
         """
         # get sample info
         sample_info = pdf.get_sample_info(
-            project_id=args[f'{arg_prefix}project_id'],
-            sample_id=args[f'{arg_prefix}sample_id']
+            project_id=args[f"{arg_prefix}project_id"],
+            sample_id=args[f"{arg_prefix}sample_id"],
         )
 
         # populate return dictionary
         ret = {
-            f'{arg_prefix}project_id': args[f'{arg_prefix}project_id'],
-            f'{arg_prefix}sample_id': args[f'{arg_prefix}sample_id'],
+            f"{arg_prefix}project_id": args[f"{arg_prefix}project_id"],
+            f"{arg_prefix}sample_id": args[f"{arg_prefix}sample_id"],
         }
 
         # get run mode
-        if f'{arg_prefix}run_mode' in args:
-            ret[f'{arg_prefix}run_mode'] = args[f'{arg_prefix}run_mode']
+        if f"{arg_prefix}run_mode" in args:
+            ret[f"{arg_prefix}run_mode"] = args[f"{arg_prefix}run_mode"]
         else:
-            run_mode_name = sample_info['run_mode'][0]
-            ret[f'{arg_prefix}run_mode'] = run_mode_name
+            run_mode_name = sample_info["run_mode"][0]
+            ret[f"{arg_prefix}run_mode"] = run_mode_name
             logger.info(f"No run_mode provided, using {run_mode_name}")
 
-        run_mode = pdf.config.get_run_mode(ret[f'{arg_prefix}run_mode'])
+        run_mode = pdf.config.get_run_mode(ret[f"{arg_prefix}run_mode"])
 
-        if f'{arg_prefix}umi_cutoff' not in args:
-            umi_cutoff = run_mode.variables['umi_cutoff'][0]
-            ret[f'{arg_prefix}umi_cutoff'] = umi_cutoff
+        if f"{arg_prefix}umi_cutoff" not in args:
+            umi_cutoff = run_mode.variables["umi_cutoff"][0]
+            ret[f"{arg_prefix}umi_cutoff"] = umi_cutoff
             logger.info(f"No umi_cutoff provided, using {umi_cutoff}")
         else:
-            ret[f'{arg_prefix}umi_cutoff'] = args[f'{arg_prefix}umi_cutoff']
+            ret[f"{arg_prefix}umi_cutoff"] = args[f"{arg_prefix}umi_cutoff"]
 
         return ret
-    
+
     ret = populate_variables_from_args(pdf, args)
 
-    if 'reference_project_id' not in args or 'reference_sample_id' not in args:
-        logger.info('No reference_project_id or reference_sample_id provided,' +
-            ' running novosparc de-novo...')
-        ret['reference_project_id'] = ''
-        ret['reference_sample_id'] = ''
-        ret['reference_umi_cutoff'] = ''
-        ret['reference_run_mode'] = ''
+    if "reference_project_id" not in args or "reference_sample_id" not in args:
+        logger.info(
+            "No reference_project_id or reference_sample_id provided,"
+            + " running novosparc de-novo..."
+        )
+        ret["reference_project_id"] = ""
+        ret["reference_sample_id"] = ""
+        ret["reference_umi_cutoff"] = ""
+        ret["reference_run_mode"] = ""
     else:
-        pdf.assert_sample(args['reference_project_id'],
-            args['reference_sample_id'])
+        pdf.assert_sample(args["reference_project_id"], args["reference_sample_id"])
 
-        logger.info("Using (project_id, sample_id)="+
-            f"({args['reference_project_id']}, {args['reference_sample_id']})" +
-            " reference, running novosparc with reference...")
+        logger.info(
+            "Using (project_id, sample_id)="
+            + f"({args['reference_project_id']}, {args['reference_sample_id']})"
+            + " reference, running novosparc with reference..."
+        )
 
-        novosparc_ret = populate_variables_from_args(pdf, args,
-            arg_prefix='reference_')
+        novosparc_ret = populate_variables_from_args(pdf, args, arg_prefix="reference_")
 
         ret = {**ret, **novosparc_ret}
 
     return ret
 
-def consolidate_pucks_merged_samples():
-    for index, row in pdf.df.iterrows():
-        project_id, sample_id = index
-        puck_ids = row['puck_barcode_file_id']
-        
-        if (not row['is_merged']) or (not pdf.is_spatial(project_id, sample_id, puck_ids)):
-            continue
-
-        if len(puck_ids) >= 1:
-            puck_ids = puck_ids[0]
-        elif len(puck_ids):
-            puck_ids = pdf.project_df_default_values['puck_barcode_file_id']
-
-        merged_from = row['merged_from']
-        puck_id_file = set()
-
-        for sample_tuple in merged_from:
-            pid = pdf.df.loc[sample_tuple]['puck_barcode_file_id']
-            pbf = pdf.df.loc[sample_tuple]['puck_barcode_file']
-            _tuple = [(id, bf) for id, bf in zip(pid, pbf)]
-            
-            puck_id_file.update([ tuple(t) for t in _tuple ])
-
-        pid, pbf = list(zip(*list(puck_id_file)))
-        pdf.df.loc[index, 'puck_barcode_file_id'] = list(pid)
-        pdf.df.loc[index, 'puck_barcode_file'] = list(pbf)
-
-
-def update_project_df_barcode_matches(prealigned=False):
-    from spacemake.snakemake.variables import puck_count_barcode_matches_summary, puck_count_prealigned_barcode_matches_summary
-
-    if prealigned:
-        _bc_file = puck_count_prealigned_barcode_matches_summary
-    else:
-        _bc_file = puck_count_barcode_matches_summary
-
-    for index, row in pdf.df.iterrows():
-        project_id, sample_id = index
-
-        puck_ids = row['puck_barcode_file_id']
-        if len(puck_ids) >= 1:
-            puck_ids = puck_ids[0]
-        elif len(puck_ids):
-            puck_ids = pdf.project_df_default_values['puck_barcode_file_id']
-
-        if (row['is_merged'] and prealigned) or (not pdf.is_spatial(project_id, sample_id, puck_ids)):
-            continue
 
-        # check if barcodes have been filtered
-        _f_barcodes_df = _bc_file.format(project_id=project_id,
-                sample_id=sample_id)
+_spacemake_instance = None
 
-        # project_df is only updated if a prealignment barcode matching file is found
-        if os.path.exists(_f_barcodes_df):
-            barcodes_df = pd.read_csv(_f_barcodes_df)
 
-            if 'pass_threshold' not in barcodes_df.columns:
-                continue
+def get_spacemake_object():
+    global _spacemake_instance
+    if _spacemake_instance is None:
+        _spacemake_instance = Spacemake(".")
 
-            above_threshold_mask = barcodes_df['pass_threshold'] == 1
+    return _spacemake_instance
 
-            _puck_barcode_files = barcodes_df[above_threshold_mask]['puck_barcode_file'].values.tolist()
-            _puck_barcode_files_id = barcodes_df[above_threshold_mask]['puck_barcode_file_id'].values.tolist()
-
-            pdf.df.at[index, 'puck_barcode_file'] = _puck_barcode_files
-            pdf.df.at[index, 'puck_barcode_file_id'] = _puck_barcode_files_id
-
-@message_aggregation(logger_name)
-def spacemake_run(pdf, args):
-    """spacemake_run.
-
-    :param pdf:
-    :param args:
-    """
-    if not os.path.isfile(config_path):
-        msg = "spacemake has not been initalised yet.\n"
-        msg += "please run `spacemake init` to start a new project"
-
-        raise SpacemakeError(msg)
-
-    samples = []
-    projects = []
-    targets = ['run_analysis']
-    with_fastqc = args.get("with_fastqc", False)
-
-    downsample = args.get("downsample", False)
-    novosparc_reconstruct = args.get('novosparc_reconstruct', False)
-
-    if downsample:
-        targets = ["downsample"]
-        samples = args.get("sample_id_list", [])
-        projects = args.get("project_id_list", [])
-
-    if novosparc_reconstruct:
-        targets = ["novosparc"]
-        novosparc_variables = get_novosparc_variables(pdf, args)
-    else:
-        novosparc_variables = {}
-
-    config_variables = {
-        "project_df": pdf.file_path,
-        "samples": samples,
-        "projects": projects,
-        "with_fastqc": with_fastqc,
-        "pwd": os.getcwd(),
-    }
-
-    # join config_variables and novosparc_variables
-    # to flatten the directory
-    config_variables = {**config_variables, **novosparc_variables}
-
-    # assert that the project_df is valid before running the pipeline
-    pdf.assert_valid()
-
-    # get the snakefile
-    snakefile = os.path.join(os.path.dirname(__file__), "snakemake/main.smk")
-    # run snakemake with prealignment filter
-    # this one runs preprocessing/alignment/counting of matching barcodes before alignment
-    preprocess_finished = snakemake.snakemake(
-        snakefile,
-        configfiles=[config_path],
-        cores=args["cores"],
-        dryrun=args["dryrun"],
-        targets=['get_stats_prealigned_barcodes', 'unload_genome_flag'],
-        touch=args["touch"],
-        force_incomplete=args["rerun_incomplete"],
-        keepgoing=args["keep_going"],
-        printshellcmds=args["printshellcmds"],
-        config=config_variables,
-    )
-    
-    if preprocess_finished is False:
-        raise SpacemakeError("an error occurred while snakemake() ran")
-
-    # update valid pucks (above threshold) before continuing to downstream
-    # this performs counting of matching barcodes after alignment
-    update_project_df_barcode_matches(prealigned=True)
-    consolidate_pucks_merged_samples()
-    pdf.dump()
-
-    # run snakemake after prealignment filter
-    preprocess_finished = snakemake.snakemake(
-        snakefile,
-        configfiles=[config_path],
-        cores=args["cores"],
-        dryrun=args["dryrun"],
-        targets=['get_whitelist_barcodes'],
-        touch=args["touch"],
-        force_incomplete=args["rerun_incomplete"],
-        keepgoing=args["keep_going"],
-        printshellcmds=args["printshellcmds"],
-        config=config_variables,
-    )
-    
-    if preprocess_finished is False:
-        raise SpacemakeError("an error occurred while snakemake() ran")
-
-    # update valid pucks (above threshold) before continuing to downstream
-    update_project_df_barcode_matches()
-    pdf.dump()
-
-    # run spacemake downstream
-    # this performs the DGE calculation, h5ad conversion, automated analysis and QC
-    analysis_finished = snakemake.snakemake(
-        snakefile,
-        configfiles=[config_path],
-        cores=args["cores"],
-        dryrun=args["dryrun"],
-        targets=targets,
-        touch=args["touch"],
-        force_incomplete=args["rerun_incomplete"],
-        keepgoing=args["keep_going"],
-        printshellcmds=args["printshellcmds"],
-        config=config_variables,
-    )
-
-    if analysis_finished is False:
-        raise SpacemakeError("an error occurred while snakemake() ran")
-
-    # at the very end dump the project_data_frame
-    pdf.dump()
-
-#################
-# DEFINE PARSER #
-#################
-# we add main parser to a dictionary
-# so that we can later call the help function
-# based on the sub-command. this is to print the
-# -h (help) functionality if no parameters are provided,
-# rather than printing an error.
-
-parser_main = argparse.ArgumentParser(
-    allow_abbrev=False,
-    description="spacemake: bioinformatic pipeline for processing and analysis of spatial-transcriptomics data",
-)
-
-parser_main.add_argument(
-    '--version',
-    action = 'store_true')
-
-parser_main_subparsers = parser_main.add_subparsers(
-        help="sub-command help",
-        dest="subcommand")
-
-parser_run = None
-parser_projects = None
-parser_config = None
-parser_init = None
-parser_spatial = None
-
-##################
-# SPACEMAKE INIT #
-##################
-parser_init = setup_init_parser(parser_main_subparsers)
-
-####################
-# SPACEMAKE CONFIG #
-####################
-## spacemake_config args
-from spacemake.config import setup_config_parser
-
-if os.path.isfile(config_path):
-    cf = ConfigFile.from_yaml(config_path)
-    # save config file
-    parser_config = setup_config_parser(cf, parser_main_subparsers)
-
-############################
-# SPACEMAKE PROJECT/SAMPLE #
-############################
-from spacemake.project_df import setup_project_parser
-
-if os.path.isfile(config_path):
-    pdf = ProjectDF(project_df, cf)
-    parser_projects = setup_project_parser(pdf, parser_main_subparsers)
-
-    #################
-    # SPACEMAKE RUN #
-    #################
-    parser_run = setup_run_parser(pdf, parser_main_subparsers)
-
-#####################
-# SPACEMAKE SPATIAL #
-#####################
-from spacemake.spatial.cmdline import setup_spatial_parser
-
-if os.path.isfile(config_path):
-    spmk = Spacemake('.')
-    parser_spatial = setup_spatial_parser(spmk, parser_main_subparsers)
-
-def cmdline():
-    import importlib.metadata
-    """cmdline."""
-    args = parser_main.parse_args()
-
-    if args.version and args.subcommand is None:
-        print(importlib.metadata.version('spacemake'))
-        return 0
-    else:
-        del args.version
-
-    parser_dict = {
-        "init": parser_init,
-        "config": parser_config,
-        "projects": parser_projects,
-        "run": parser_run,
-        "main": parser_main,
-        "spatial": parser_spatial
-    }
-    # get the function to be run
-    if "func" in args:
-        func = args.func
-    # else print help
-    else:
-        if args.subcommand is not None:
-            parser_dict[args.subcommand].print_help()
-        else:
-            parser_dict["main"].print_help()
-        return 0
 
-    # get the args and delete the func key, get only set values
-    args = {key: value for key, value in vars(args).items() if value is not None}
-    args.pop("func", None)
-    # pop also main,
-    args.pop("subcommand", None)
+# def get_ConfigFile():
+#     spmk = get_spacemake_object()
+#     return spmk.config
 
-    func(args)
 
-if __name__ == "__main__":
-    cmdline()
+# def get_ProjectDF():
+#     spmk = get_spacemake_object()
+#     return spmk.project_df
diff --git a/spacemake/snakemake/longread.smk b/spacemake/snakemake/longread.smk
index a4ba2a12..e92c1bd5 100644
--- a/spacemake/snakemake/longread.smk
+++ b/spacemake/snakemake/longread.smk
@@ -32,7 +32,7 @@ lr_overview_csv = lr_overview_dir + 'overview.csv'
 LR_RAW_FILES = {}
 LR_SIGNATURE = {}
 LR_REPORT_STATS = []
-def get_longread_output():
+def get_longread_output(project_df=None, config=None, **kw):
     """
     This function is called from main.smk at least once 
     to determine which output files need to be generated
diff --git a/spacemake/snakemake/main.smk b/spacemake/snakemake/main.smk
index f0e49273..d6d0dd0a 100644
--- a/spacemake/snakemake/main.smk
+++ b/spacemake/snakemake/main.smk
@@ -1,7 +1,6 @@
 #########
 # about #
 #########
-__version__ = '0.1.1'
 __author__ = ['Nikos Karaiskos', 'Tamas Ryszard Sztanka-Toth']
 __license__ = 'GPL'
 __email__ = ['nikolaos.karaiskos@mdc-berlin.de', 'tamasryszard.sztanka-toth@mdc-berlin.de']
@@ -15,9 +14,10 @@ import numpy as np
 import math
 import scanpy as sc
 
-from spacemake.preprocess import dge_to_sparse_adata, attach_barcode_file,\
+from spacemake.preprocess.dge import dge_to_sparse_adata, attach_barcode_file,\
     parse_barcode_file, load_external_dge, attach_puck_variables, attach_puck
-from spacemake.spatial import create_meshed_adata, puck_collection
+from spacemake.spatial.util import create_meshed_adata
+import spacemake.spatial.puck_collection as puck_collection
 from spacemake.project_df import ProjectDF
 from spacemake.config import ConfigFile
 from spacemake.errors import SpacemakeError
@@ -45,6 +45,9 @@ global_tmp = config['temp_dir']
 repo_dir = os.path.dirname(workflow.snakefile)
 spacemake_dir = os.path.dirname(os.path.dirname(workflow.snakefile))
 
+import logging
+smk_logger = logging.getLogger("spacemake.main.smk")
+
 #######################
 # DIRECTORY STRUCTURE #
 #######################
@@ -78,8 +81,8 @@ def register_module_output_hook(hook, module="built-in"):
 def get_module_outputs():
     outputs = []
     for hook, module in _module_output_hooks:
-        for out in hook():
-            print(f"output provided by '{module}' module (via '{hook.__name__}'): '{out}'")
+        for out in hook(project_df=project_df, config=config):
+            smk_logger.debug(f"output provided by '{module}' module (via '{hook.__name__}'): '{out}'")
             outputs.append(out)
     
     return outputs
@@ -106,20 +109,20 @@ if "custom_rules" in config:
 
 # global wildcard constraints
 wildcard_constraints:
-    umi_cutoff = '\d+',
-    dge_cleaned='|\.cleaned',
-    dge_type = '|'.join(dge_types),
-    pacbio_ext = 'fq|fastq|bam',
-    polyA_adapter_trimmed = '|\.polyA_adapter_trimmed',
-    mm_included = '|\.mm_included',
-    n_beads = '[0-9]+|spatial|external',
-    is_external = '|\.external',
-    spot_diameter_um = '[0-9]+',
-    spot_distance_um = '[0-9]+|hexagon',
-    data_root_type = 'complete_data|downsampled_data',
-    downsampling_percentage = '\/[0-9]+|',
-    puck_barcode_file_id = '(?!puck_collection)[^.]+',
-    puck_barcode_file_id_qc = '[^.]+'
+    umi_cutoff = r'\d+',
+    dge_cleaned=r'|\.cleaned',
+    dge_type = r'|'.join(dge_types),
+    pacbio_ext = r'fq|fastq|bam',
+    polyA_adapter_trimmed = r'|\.polyA_adapter_trimmed',
+    mm_included = r'|\.mm_included',
+    n_beads = r'[0-9]+|spatial|external',
+    is_external = r'|\.external',
+    spot_diameter_um = r'[0-9]+',
+    spot_distance_um = r'[0-9]+|hexagon',
+    data_root_type = r'complete_data|downsampled_data',
+    downsampling_percentage = r'\/[0-9]+|',
+    puck_barcode_file_id = r'(?!puck_collection)[^.]+',
+    puck_barcode_file_id_qc = r'[^.]+'
 
 #############
 # Main rule #
@@ -172,7 +175,10 @@ rule get_whitelist_barcodes:
             downsampling_percentage='', run_on_external=False),
         get_output_files(puck_barcode_files_summary,
             data_root_type = 'complete_data',
-            downsampling_percentage='', run_on_external=False)
+            downsampling_percentage='', run_on_external=False),
+        # get_output_files(parsed_spatial_barcodes,
+        #     data_root_type = 'complete_data',
+        #     downsampling_percentage='', run_on_external=False)
 
 ##############
 # DOWNSAMPLE #
@@ -274,33 +280,21 @@ rule tag_reads_bc_umi:
         bc = lambda wildcards: get_bc_preprocess_settings(wildcards)
     output:
         assigned = tagged_bam,
-        unassigned = unassigned,
-        bc_stats = reverse_reads_mate_1.replace(reads_suffix, ".bc_stats.tsv")
+        log = tagged_bam_log
     log:
         reverse_reads_mate_1.replace(reads_suffix, ".preprocessing.log")
-    threads: 4
+    threads: max(min(workflow.cores * 0.5, 16), 1)
     shell:
-        "python {spacemake_dir}/preprocess/cmdline.py "
+        "python {spacemake_dir}/bin/fastq_to_uBAM.py "
         "--sample={wildcards.sample_id} "
         "--read1={input.R1} "
         "--read2={input.R2} "
         "--parallel={threads} "
-        "--save-stats={output.bc_stats} "
-        "--log-file={log} "
-        "--bc1-ref={params.bc.bc1_ref} "
-        "--bc2-ref={params.bc.bc2_ref} "
-        "--bc1-cache={params.bc.bc1_cache} "
-        "--bc2-cache={params.bc.bc2_cache} "
-        "--threshold={params.bc.score_threshold} "
+	    "--out-bam={output.assigned} "
         "--cell='{params.bc.cell}' "
-        "--cell-raw='{params.bc.cell_raw}' "
-        "--out-format=bam "
-        "--out-unassigned={output.unassigned} "
-        "--out-assigned=/dev/stdout "
         "--UMI='{params.bc.UMI}' "
         "--bam-tags='{params.bc.bam_tags}' "
-        "--min-opseq-score={params.bc.min_opseq_score} "
-        "| samtools view -bh /dev/stdin > {output.assigned} "
+        "--log-file='{output.log}' "
 
 rule run_fastqc:
     input:
@@ -325,38 +319,48 @@ rule get_barcode_readcounts:
     input:
         unpack(get_final_bam)
     output:
-        barcode_readcounts
+        barcode_readcounts,
+        barcode_readcounts_log
     params:
         cell_barcode_tag = lambda wildcards: get_bam_tag_names(
             project_id = wildcards.project_id,
             sample_id = wildcards.sample_id)['{cell}']
+    threads: max(min(workflow.cores * 0.5, 16), 1)
     shell:
+        # {dropseq_tools}/BamTagHistogram -m 32g 
         """
-        {dropseq_tools}/BamTagHistogram -m 32g \
-        I= {input} \
-        O= {output} \
-        TAG={params.cell_barcode_tag} \
-        READ_MQ=0
+        python {spacemake_dir}/bin/BamTagHistogram.py \
+        --input {input} \
+        --output {output[0]} \
+        --tag {params.cell_barcode_tag} \
+        --min-count 1 \
+        --log-file {output[1]} \
         """
+        #READ_MQ=0
 
 rule get_barcode_readcounts_prealigned:
     # we perform some preliminary counting on the prealigned reads
     input:
         tagged_bam
     output:
-        barcode_readcounts_prealigned
+        barcode_readcounts_prealigned,
+        barcode_readcounts_prealigned_log
     params:
         cell_barcode_tag = lambda wildcards: get_bam_tag_names(
             project_id = wildcards.project_id,
             sample_id = wildcards.sample_id)['{cell}']
+    threads: max(min(workflow.cores * 0.5, 16), 1)
     shell:
+        # {dropseq_tools}/BamTagHistogram -m 32g 
         """
-        {dropseq_tools}/BamTagHistogram -m 32g \
-        I= {input} \
-        O= {output} \
-        TAG={params.cell_barcode_tag} \
-        READ_MQ=0
+        python {spacemake_dir}/bin/BamTagHistogram.py \
+        --input {input} \
+        --output {output[0]} \
+        --tag {params.cell_barcode_tag} \
+        --min-count 1 \
+        --log-file {output[1]} \
         """
+        # READ_MQ=0
 
 rule merge_stats_prealigned_spatial_barcodes:
     input:
@@ -378,11 +382,11 @@ rule merge_stats_prealigned_spatial_barcodes:
     shell:
         "python {spacemake_dir}/snakemake/scripts/n_intersect_sequences.py"
         " --query {input.bc_prealign}"
-        " --query-plain-skip 0"
+        " --query-plain-skip 1"
         " --query-plain-column 1"
         " --target {input.puck_barcode_files}"
         " --target-id {params.pbc_id}"
-        " --target-plain-column 0"
+        " --target-column 'cell_bc'"
         " --summary-output {output}"
         " --min-threshold {params.min_threshold}"
         " --n-jobs {threads}"   
@@ -408,28 +412,20 @@ rule create_spatial_barcode_file:
         unpack(get_puck_file),
         unpack(get_all_barcode_readcounts)
     output:
-        parsed_spatial_barcodes
-    run:
-        # TODO: benchmark this rule - set+np index instead of merge?
-        # load all readcounts
-        bc_readcounts=[pd.read_table(bc_rc, skiprows=1,
-            names=['read_n', 'cell_bc']) for bc_rc in input['bc_readcounts']]
-
-        # join them together
-        bc_readcounts = pd.concat(bc_readcounts)
-
-        # remove duplicates
-        bc_readcounts.drop_duplicates(subset='cell_bc', keep='first',
-            inplace=True)
-
-        # load barcode file and parse it
-        bc = parse_barcode_file(input[0])
-        bc.reset_index(level=0, inplace=True)
-        # inner join to get rid of barcode without any data
-        bc = pd.merge(bc, bc_readcounts, how='inner', on='cell_bc')
-        bc = bc[['cell_bc', 'x_pos', 'y_pos']]
-
-        bc.to_csv(output[0], index=False)
+        parsed_spatial_barcodes,
+        temp(parsed_spatial_barcodes_summary)
+    shell:
+        "python {spacemake_dir}/snakemake/scripts/n_intersect_sequences.py"
+        " --query {input.bc_readcounts}"
+        " --query-plain-skip 1"
+        " --query-plain-column 1"
+        " --target {input.barcode_file}"
+        " --target-id {wildcards.puck_barcode_file_id}"
+        " --target-column 'cell_bc'"
+        " --output {output[0]}"
+        " --summary-output {output[1]}"
+        " --n-jobs {threads}"   
+        " --chunksize 10000000"
 
 rule create_spatial_barcode_whitelist:
     input: parsed_spatial_barcodes
@@ -747,121 +743,48 @@ rule split_reads_sam_to_bam:
     shell:
         "sambamba view -S -h -f bam -t {threads} -o {output} {input}"
 
+
 rule count_barcode_matches:
     input:
-        unpack(get_barcode_files_matching_summary_input)
+        unpack(get_barcode_summary_files_matching_summary_input)
     output:
-        puck_count_barcode_matches_summary
+        puck_count_barcode_matches_summary,
+        puck_barcode_files_summary
     params:
-        pbf_ids = lambda wildcards: project_df.get_puck_barcode_ids_and_files(
-            project_id = wildcards.project_id,
-            sample_id = wildcards.sample_id)[0],
         run_mode_variables = lambda wildcards:
             project_df.config.get_run_mode(list(get_run_modes_from_sample(
-            wildcards.project_id, wildcards.sample_id).keys())[0]).variables
+            wildcards.project_id, wildcards.sample_id).keys())[0]).variables,
+        puck_variables = lambda wildcards:
+            project_df.get_puck_variables(wildcards.project_id, wildcards.sample_id,
+            return_empty=True)
     run:
         import os
-        out_df = pd.DataFrame(columns=[
-            'puck_barcode_file_id',
-            'puck_barcode_file',
-            'parsed_barcode_file',
-            'n_barcodes',
-            'n_matching',
-            'matching_ratio'
-            ])
-
-        if ('puck_barcode_files' in input.keys() and
-            'parsed_spatial_barcode_files' in input.keys()):
-            for pbf_id, pbf, parsed_barcode_file in zip(
-                    params['pbf_ids'],
-                    input['puck_barcode_files'],
-                    input['parsed_spatial_barcode_files'],
-                ):
-                n_barcodes = pd.read_csv(pbf).shape[0]
-                n_matching = pd.read_csv(parsed_barcode_file).shape[0]
-                matching_ratio = round(float(n_matching)/n_barcodes, 2)
-                
-                out_df = pd.concat([out_df, pd.DataFrame({
-                    'puck_barcode_file_id': [pbf_id],
-                    'puck_barcode_file': [pbf],
-                    'parsed_barcode_file': [parsed_barcode_file],
-                    'n_barcodes': [n_barcodes],
-                    'n_matching': [n_matching],
-                    'matching_ratio': [matching_ratio],
-                })], ignore_index=True, sort=False)
+        out_df = pd.DataFrame( {
+            'puck_barcode_file_id': [],
+            'puck_barcode_file': [], 
+            'parsed_barcode_file': [],
+            'n_barcodes': [],
+            'n_matching': [],
+            'matching_ratio': [],
+            'pass_threshold': [],
+            'x_pos_min_px': [],
+            'x_pos_max_px': [],
+            'y_pos_min_px': [],
+            'y_pos_max_px': [],
+            'px_by_um': []
+        })
+
+        if 'matched_barcode_files_summary' in input.keys():
+            for matched_barcode_file_summary in input['matched_barcode_files_summary']:
+                pbf_df = pd.read_csv(matched_barcode_file_summary)
+                out_df = pd.concat([out_df, pbf_df])
 
             # we use > so whenever default: 0 we exclude empty pucks
             above_threshold_mask = out_df.matching_ratio > params['run_mode_variables']['spatial_barcode_min_matches']
             out_df['pass_threshold'] = 0
+            out_df['px_by_um'] = (out_df['x_pos_max_px'] - out_df['x_pos_min_px'])  / params['puck_variables']['width_um']
             out_df['pass_threshold'][above_threshold_mask] = 1
 
-        out_df.to_csv(output[0], index=False)
-
-
-rule create_barcode_files_matching_summary:
-    input:
-        unpack(get_barcode_files_matching_summary_input),
-        puck_count_barcode_matches_summary
-    output:
-        puck_barcode_files_summary
-    params:
-        pbf_ids = lambda wildcards: project_df.get_puck_barcode_ids_and_files(
-            project_id = wildcards.project_id,
-            sample_id = wildcards.sample_id)[0],
-        puck_variables = lambda wildcards:
-            project_df.get_puck_variables(wildcards.project_id, wildcards.sample_id,
-                return_empty=True),
-        run_mode_variables = lambda wildcards:
-            project_df.config.get_run_mode(list(get_run_modes_from_sample(
-            wildcards.project_id, wildcards.sample_id).keys())[0]).variables
-    run:
-        import os
-        out_df = pd.DataFrame(columns=[
-            'puck_barcode_file_id',
-            'puck_barcode_file',
-            'parsed_barcode_file',
-            'n_barcodes',
-            'n_matching',
-            'matching_ratio', 
-            'x_pos_min_px',
-            'x_pos_max_px',
-            'y_pos_min_px',
-            'y_pos_max_px',
-            'px_by_um'])
-
-        if ('puck_barcode_files' in input.keys() and
-            'parsed_spatial_barcode_files' in input.keys()):
-            for pbf_id, pbf, parsed_barcode_file in zip(
-                    params['pbf_ids'],
-                    input['puck_barcode_files'],
-                    input['parsed_spatial_barcode_files'],
-                ):
-                pbf_df = parse_barcode_file(pbf)
-                n_barcodes = pbf_df.shape[0]
-                n_matching = pd.read_csv(parsed_barcode_file).shape[0]
-                matching_ratio = round(float(n_matching)/n_barcodes, 2)
-
-                # calculate puck metrics
-                x_pos_min_px = pbf_df.x_pos.min()
-                x_pos_max_px = pbf_df.x_pos.max()
-                y_pos_min_px = pbf_df.y_pos.min()
-                y_pos_max_px = pbf_df.y_pos.max()
-
-                px_by_um = (x_pos_max_px - x_pos_min_px) 
-                px_by_um = px_by_um / params['puck_variables']['width_um']
-                
-                out_df = pd.concat([out_df, pd.DataFrame({
-                    'puck_barcode_file_id': [pbf_id],
-                    'puck_barcode_file': [pbf],
-                    'parsed_barcode_file': [parsed_barcode_file],
-                    'n_barcodes': [n_barcodes],
-                    'n_matching': [n_matching],
-                    'matching_ratio': [matching_ratio],
-                    'x_pos_min_px': [x_pos_min_px],
-                    'x_pos_max_px': [x_pos_max_px],
-                    'y_pos_min_px': [y_pos_min_px],
-                    'y_pos_max_px': [y_pos_max_px],
-                    'px_by_um': [px_by_um],
-                })], ignore_index=True, sort=False)
-
-        out_df.to_csv(output[0], index=False)
\ No newline at end of file
+        out_df[['puck_barcode_file_id', 'puck_barcode_file', 'parsed_barcode_file', 'n_barcodes', 'n_matching', 'matching_ratio', 'pass_threshold']].to_csv(output[0], index=False)
+        out_df[['puck_barcode_file_id','puck_barcode_file','parsed_barcode_file','n_barcodes','n_matching','matching_ratio','x_pos_min_px','x_pos_max_px','y_pos_min_px','y_pos_max_px','px_by_um']].to_csv(output[1], index=False)
+        
diff --git a/spacemake/snakemake/mapping.smk b/spacemake/snakemake/mapping.smk
index 454266f1..2b62fc63 100644
--- a/spacemake/snakemake/mapping.smk
+++ b/spacemake/snakemake/mapping.smk
@@ -11,6 +11,29 @@ The module takes over after pre-processing is done and hands over a "final.bam"
 to all downstream steps (DGE generation, qc_sheets, ...).
 """
 
+# This is where all the python functions and string constants
+# live
+from spacemake.map_strategy import *
+
+# used to fetch all info needed to create a BAM file
+MAP_RULES_LKUP = map_data['MAP_RULES_LKUP']
+#   key: path of output BAM file
+#   value: dotdict with plenty of attributes
+
+# used for symlink name to source mapping
+BAM_SYMLINKS = map_data['BAM_SYMLINKS']
+
+# used for automated mapping index generation
+INDEX_FASTA_LKUP = map_data['INDEX_FASTA_LKUP']
+#   key: bt2_index_file or star_index_file
+#   value: dotdict with 
+#       .ref_path: path to genome FASTA
+#       .ann_path: path to annotation file (GTF) [optional]
+
+# needed for later stages of SPACEMAKE which require one "star.Log.final.out" file.
+STAR_FINAL_LOG_SYMLINKS = map_data['STAR_FINAL_LOG_SYMLINKS']
+
+register_module_output_hook(get_mapped_BAM_output, "mapping.smk")
 #####################################
 #### snakemake string templates #####
 #####################################
@@ -53,25 +76,6 @@ bt2_index_file = bt2_index_param + '.1.bt2'
 species_reference_sequence = 'species_data/{species}/{ref_name}/sequence.fa'
 species_reference_annotation = 'species_data/{species}/{ref_name}/annotation.gtf'
 
-# used to fetch all info needed to create a BAM file
-MAP_RULES_LKUP = {}
-#   key: path of output BAM file
-#   value: dotdict with plenty of attributes
-
-# used for symlink name to source mapping
-BAM_SYMLINKS = {}
-
-# used for automated mapping index generation
-INDEX_FASTA_LKUP = {}
-#   key: bt2_index_file or star_index_file
-#   value: dotdict with 
-#       .ref_path: path to genome FASTA
-#       .ann_path: path to annotation file (GTF) [optional]
-
-# needed for later stages of SPACEMAKE which require one "star.Log.final.out" file.
-STAR_FINAL_LOG_SYMLINKS = {}
-# We create a symlink from the mapping that was accompanied by the 'final' identifier
-
 default_BT2_MAP_FLAGS = (
     " --local"
     " -L 10 -D 30 -R 30"
@@ -94,142 +98,20 @@ default_STAR_MAP_FLAGS = (
     " --limitOutSJcollapsed 5000000"
 )
 
+# TODO: port remaining python code to map_strategy.py
+# to expose it to enable coverage analysis and unit-testing
+# possibly best way is to turn map_strategy into a class-instance that can be set-up with 
+# project_df and config and make all these utility functions methods.
+# that would also remove the need for map_data which is already an attempt to mitigate global
+# variables.
 
-######################################################
-##### Utility functions specific for this module #####
-######################################################
-
-def wc_fill(x, wc):
-    """
-    Versatile helper function that can render any string template used in the mapping module,
-    either filling in from a Wildcards object, or from a dotdict.
-    """
-    return x.format(
-        sample_id=getattr(wc, "sample_id", "NO_sample_id"),
-        project_id=getattr(wc, "project_id", "NO_project_id"),
-        species=getattr(wc, "species", "NO_species"),
-        ref_name=getattr(wc, "ref_name", "NO_ref_name"),
-        mapper=getattr(wc, "mapper", "NO_mapper"),
-        link_name=getattr(wc, "link_name", "NO_link_name"),
-        polyA_adapter_trimmed=getattr(wc, "polyA_adapter_trimmed", "")
-    )
-
-def mapstr_to_targets(mapstr, left="uBAM", final="final"):
-    """
-    Converts a mapping strategy provided as a string into a series of map rules, which translate into 
-    BAM names and their dependencies. Downstream, rule matching is guided by the convention of the 
-    strategy-defined BAM filenames ending in "STAR.bam" or "bowtie2.bam".
-
-    Examples:
-
-        bowtie2:rRNA->STAR:genome:final
-
-    The input to the first mappings is always the uBAM - the CB and UMI tagged, pre-processed, 
-    but unmapped reads.
-    map-rules are composed of two or three paramters, separated by ':'.
-    The first two parameters for the mapping are <mapper>:<reference>. The target BAM will have 
-    the name <reference>.<mapper>.bam.
-    Optionally, a triplet can be used <mapper>:<reference>:<symlink> where the presence of <symlink> 
-    indicates that the BAM file should additionally be made accessible under the name <symlink>.bam, a useful 
-    shorthand or common hook expected by other stages of SPACEMAKE. A special symlink name is "final"
-    which is required by downstream stages of SPACEMAKE. If no "final" is specified, the last map-rule
-    automatically is selected and symlinked as "final".
-    Note, due to the special importance of "final", it may get modified to contain other flags of the run-mode
-    that are presently essential for SPACEMAKE to have in the file name ("final.bam" may in fact be 
-    "final.polyA_adapter_trimmed.bam")
-
-    The example above is going to create
-        
-        (1) rRNA.bowtie2.bam
-        
-            using bowtie2 and the index associated with the "rRNA" reference
-
-
-        (2) genome.STAR.bam 
-        
-            using STAR on the *unmapped* reads from BAM (1)
-
-
-        (3) final..bam
-        
-            a symlink pointing to the actual BAM created in (2).
-
-    Note that one BAM must be designated 'final.bam', or the last BAM file created will be selected as final.
-    (used as input to downstream processing rules for DGE creation, etc.)
-
-    NOTE: Parallel mappings can be implemented by using commata:
-
-        bowtie2:rRNA:rRNA,STAR:genome:final
-
-        This rule differs from the first example because it will align the unmapped reads from the uBAM
-        in parallel to the rRNA reference and to the genome. In this way the same reads can match to both
-        indices.
-
-    NOTE: Gene tagging will be applied automatically if annotation data were provided for the associated 
-    reference index (by using spacemake config add_species --annotation=... )
-    """
-    def process(token, left):
-        """
-        based on the left-hand side name (left) and the rule encoded in token,
-        create one mapping rule and up to one symlink rule.
-        """
-
-        parts = token.split(":")
-        mr = dotdict()
-        lr = None
-
-        if len(parts) == 2:
-            mapper, ref = parts
-            link_name = None
-        elif len(parts) == 3:
-            mapper, ref, link_name = parts
-            link_name = link_name.replace("final", final)
-        else:
-            raise ValueError(f"map_strategy contains a map-rule with unexpected number of parameters: {parts}")
-
-        mr.input_name = left
-        mr.mapper = mapper
-        mr.ref_name = ref
-        mr.out_name = f"{ref}.{mapper}"
-
-        if link_name:
-            lr = dotdict(link_src=mr.out_name, link_name=link_name, ref_name=mr.ref_name)
-
-        return mr, lr
-
-
-    map_rules = []
-    link_rules = []
-
-    chain = mapstr.split("->")
-    final_link = None
-
-    while chain:
-        # print("chain:", chain)
-        right = chain.pop(0)
-        # print(f"left='{left}' right='{right}'")
-        if left == right:
-            continue
-
-        for r in right.split(","):
-            mr, lr = process(r, left=left)
-            map_rules.append(mr)
-            
-            if lr:
-                link_rules.append(lr)
-                # check if we have a "final" mapping
-                if final in lr.link_name:
-                    final_link = lr
-
-        left = mr.out_name
-
-    if not final_link:
-        # we need to manufacture a "final" link_rule by taking the last mapping
-        last = map_rules[-1]
-        link_rules.append(dotdict(link_src=last.out_name, link_name=final, ref_name=last.ref_name))
-
-    return map_rules, link_rules
+def is_paired_end(project_id, sample_id):
+    row = project_df.df.loc[(project_id, sample_id)].to_dict()
+    #print(f"{project_id} {sample_id} -> {row['barcode_flavor']}")
+    flavor_d = project_df.config.get_variable("barcode_flavors", name=row['barcode_flavor'])
+    pe = flavor_d.get('paired_end', 'single-end')
 
+    return pe != 'single-end'
 
 def get_star_flag(flag, default_strategy="STAR:genome:final"):
     out_files = []
@@ -249,96 +131,6 @@ def get_star_flag(flag, default_strategy="STAR:genome:final"):
 
     return set(out_files)
 
-
-
-def get_mapped_BAM_output(default_strategy="STAR:genome:final"):
-    """
-    This function is called from main.smk at least once 
-    to determine which output BAM files need to be generated and 
-    to parse the map_strategy into rules and dependencies.
-    """
-    out_files = []
-
-    for index, row in project_df.df.iterrows():
-        # rules so far are "local" to each sample. Here we create full paths, merging with
-        # project_id/sample_id from the project_df
-
-        map_strategy = getattr(row, "map_strategy", default_strategy)
-        map_rules, link_rules = mapstr_to_targets(map_strategy, left=ubam_input, final=final_target)
-
-        species_d = project_df.config.get_variable("species", name=row.species)
-        is_merged = project_df.get_metadata(
-            "is_merged", project_id=index[0], sample_id=index[1]
-        )
-        if is_merged:
-            continue
-
-        for mr in map_rules:
-            mr.project_id = index[0]
-            mr.sample_id = index[1]
-            mr.species = row.species
-            
-            mr.out_path = wc_fill(mapped_bam, mr)
-            
-            mr.link_name = mr.input_name
-            mr.input_path = wc_fill(linked_bam, mr)
-
-            mr.ref_path = species_d[mr.ref_name]["sequence"]
-            mr.ann_path = species_d[mr.ref_name].get("annotation", None)
-            if mr.ann_path:
-                mr.ann_final = wc_fill(species_reference_annotation, mr)
-            else:
-                mr.ann_final = []
-
-            default_STAR_INDEX = wc_fill(star_index, mr)
-            default_BT2_INDEX = wc_fill(bt2_index_param, mr)
-            if mr.mapper == "bowtie2":
-                mr.map_flags = species_d[mr.ref_name].get("BT2_flags", default_BT2_MAP_FLAGS)
-                mr.map_index_param = species_d[mr.ref_name].get("BT2_index", default_BT2_INDEX) # the parameter passed on to the mapper
-                mr.map_index = os.path.dirname(mr.map_index_param) # the index_dir
-                mr.map_index_file = mr.map_index_param + ".1.bt2" # file present if the index is actually there
-
-            elif mr.mapper == "STAR":
-                mr.map_flags = species_d[mr.ref_name].get("STAR_flags", default_STAR_MAP_FLAGS)
-                mr.map_index = species_d[mr.ref_name].get("index_dir", default_STAR_INDEX)
-                mr.map_index_param = mr.map_index
-                mr.map_index_file = mr.map_index + "/SAindex"
-
-            MAP_RULES_LKUP[mr.out_path] = mr
-            INDEX_FASTA_LKUP[mr.map_index_file] = mr
-            #out_files.append(mr.out_path)
-
-        # process all symlink rules
-        for lr in link_rules:
-            lr.link_path = linked_bam.format(project_id=index[0], sample_id=index[1], link_name=lr.link_name)
-            lr.src_path = linked_bam.format(project_id=index[0], sample_id=index[1], link_name=lr.link_src)
-            BAM_SYMLINKS[lr.link_path] = lr.src_path
-
-            if lr.link_name == final_target:
-                final_log_name = star_log_file.format(project_id=index[0], sample_id=index[1])
-                final_log = star_target_log_file.format(ref_name=lr.ref_name, project_id=index[0], sample_id=index[1])
-                # print("STAR_FINAL_LOG_SYMLINKS preparation", target, src, final_log_name, "->", final_log)
-                STAR_FINAL_LOG_SYMLINKS[final_log_name] = final_log
-                
-                out_files.append(lr.link_path)
-
-    # for k,v in sorted(MAP_RULES_LKUP.items()):
-    #     print(f"map_rules for '{k}'")
-    #     print(v)
-
-    # print("BAM_SYMLINKS")
-    # for k, v in BAM_SYMLINKS.items():
-    #     print(f"    output={k} <- source={v}")
-
-    # print("out_files", out_files)
-    return out_files
-
-register_module_output_hook(get_mapped_BAM_output, "mapping.smk")
-
-#############################################
-#### utility functions used by the rules ####
-#############################################
-
 def get_species_reference_info(species, ref):
     refs_d = project_df.config.get_variable("species", name=species)
     return refs_d[ref]
@@ -350,7 +142,7 @@ def get_reference_attr_from_wc(wc, attr):
 def get_map_rule(wc):
     output = wc_fill(mapped_bam, wc)
     # print(f"get_map_rules output={output}")
-    return MAP_RULES_LKUP[output]
+    return map_data['MAP_RULES_LKUP'][output]
 
 def get_map_inputs(wc, mapper="STAR"):
     wc = dotdict(wc.items())
@@ -386,7 +178,6 @@ def get_map_params(wc, output, mapper="STAR"):
         'flags' : mr.map_flags,
     }
 
-
 ##############################################################################
 #### Snakemake rules for mapping, symlinks, and mapping-index generation #####
 ##############################################################################
@@ -598,7 +389,7 @@ rule unload_genome_flag:
 
 rule unload_genome:
     input:
-        bams=ancient(get_mapped_BAM_output()),
+        bams=ancient(get_mapped_BAM_output(project_df)),
         index_dir=star_index, # we put last so it is accessible
     output:
         temp(touch(star_index_unloaded)),
diff --git a/spacemake/snakemake/scripts/automated_analysis.py b/spacemake/snakemake/scripts/automated_analysis.py
index 1c8d83e1..8ba9c0ad 100644
--- a/spacemake/snakemake/scripts/automated_analysis.py
+++ b/spacemake/snakemake/scripts/automated_analysis.py
@@ -3,7 +3,7 @@
 import scanpy as sc
 import squidpy as sq
 
-from spacemake.spatial import detect_tissue
+from spacemake.spatial.util import detect_tissue
 
 # expect fitlered .h5ad dge, with spatial coords attached, tissue detected
 adata = sc.read_h5ad(snakemake.input[0])
diff --git a/spacemake/snakemake/scripts/create_spatial_barcode_file.py b/spacemake/snakemake/scripts/create_spatial_barcode_file.py
new file mode 100644
index 00000000..b037b6c4
--- /dev/null
+++ b/spacemake/snakemake/scripts/create_spatial_barcode_file.py
@@ -0,0 +1,331 @@
+import logging
+
+import multiprocessing as mp
+import pandas as pd
+import time
+
+from spacemake.util import message_aggregation, FASTQ_src
+
+"""
+Sequence intersection between a query file and target files (pucks).
+
+Usage:
+    python create_spatial_barcode_file.py \
+        --query input_reads.txt \
+        --target puck_file1.txt puck_file2.txt \
+        --output puck_output1.txt puck_output2.txt
+
+Author:
+    Daniel León-Periñán
+"""
+
+logger_name = "spacemake.snakemake.scripts.create_spatial_barcode_file"
+logger = logging.getLogger(logger_name)
+AVAILABLE_CPUS = mp.cpu_count()
+
+
+def setup_parser(parser):
+    """
+    Set up command-line arguments for the script.
+
+    :param parser: Argument parser object.
+    :type parser: argparse.ArgumentParser
+    :returns: Updated argument parser object.
+    :rtype: argparse.ArgumentParser
+    """
+    parser.add_argument(
+        "--query",
+        type=str,
+        help="query reads file for lookup into 'target'(s) database",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--query-tag",
+        type=str,
+        help="if input is a bam file, the specified tag will be used as the sequence",
+        default="CB",
+    )
+
+    parser.add_argument(
+        "--query-plain-skip",
+        type=int,
+        help="number of lines to skip, if target input is a plain text file",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--query-plain-column",
+        type=int,
+        help="column to parse from query, if input is a plain text file",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--query-separator",
+        type=str,
+        default="\t",
+        help="if input is a tabular plain text file, this specifies the character for column separation",
+        choices=["\t", ",", ";"],
+    )
+
+    parser.add_argument(
+        "--target",
+        type=str,
+        nargs="+",
+        help="target puck files against which search is performed",
+        required=True,
+    )
+
+    parser.add_argument(
+        "--target-id",
+        type=str,
+        nargs="+",
+        help="target puck ids for summary output",
+        required=True,
+    )
+
+    parser.add_argument(
+        "--target-tag",
+        type=str,
+        help="if input is a bam file, the specified tag will be used as the sequence",
+        default="CB",
+    )
+
+    parser.add_argument(
+        "--target-plain-skip",
+        type=int,
+        help="number of lines to skip, if target input is a plain text file",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--target-plain-column",
+        type=int,
+        help="column to parse from target, if input is a plain text file",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--target-separator",
+        type=str,
+        default="\t",
+        help="if input is a tabular plain text file, this specifies the character for column separation",
+        choices=["\t", ",", ";"],
+    )
+
+    parser.add_argument(
+        "--summary-output",
+        type=str,
+        help="a summary output file containing the number of matches between query and target, per target file",
+        required=True,
+    )
+
+    parser.add_argument(
+        "--min-threshold",
+        type=float,
+        default=0,
+        help="targets with 'matching_ratio' above --min-threshold are marked as True; else, False.",
+        required=False,
+    )
+
+    parser.add_argument(
+        "--n-jobs",
+        type=int,
+        help="number of parallel jobs (up to {AVAILABLE_CPUS} in this machine) for database lookup",
+        metavar=f"[0-{AVAILABLE_CPUS-1}]",
+        choices=range(0, AVAILABLE_CPUS),
+        default=1,
+    )
+
+    return parser
+
+
+def BAM_src(src: str, tag: str = None) -> tuple:
+    """
+    Generate read data from a BAM file.
+
+    :param src: Path to the BAM file.
+    :type src: str
+    :param tag: Tag to be used as the sequence. Defaults to None.
+    :type tag: str, optional
+    :yields: Tuple containing read name, sequence, and quality.
+    :rtype: tuple
+    """
+
+    import pysam
+
+    bam = pysam.AlignmentFile(src, "rb", check_sq=False)
+    for read in bam.fetch(until_eof=True):
+        if tag is None:
+            yield read.query_name, read.query_sequence, read.query_qualities
+        else:
+            yield None, read.get_tag(tag), None
+
+
+def plain_src(f: str, skip: int = 0, column: int = 0, separator: str = "\t") -> tuple:
+    """
+    Generate read data from a plain text file.
+
+    :param f: Path to the plain text file.
+    :type f: str
+    :param skip: Number of lines to skip. Defaults to 0.
+    :type skip: int, optional
+    :param column: Column to parse from the file. Defaults to 0.
+    :type column: int, optional
+    :param separator: Character for column separation. Defaults to "\t".
+    :type separator: str, optional
+    :yields: Tuple containing read name, sequence, and quality.
+    :rtype: tuple
+    """
+    
+    reads = pd.read_csv(f, sep=separator, skiprows=skip)
+    reads = reads.iloc[:, column]
+
+    for read in reads:
+        yield None, read, None
+
+
+def read_with_tag(
+    f: str, tag: str = None, skip: int = None, column: int = 0, separator: str = "\t"
+) -> str:  
+    """
+    Read data from various sources based on the file type.
+
+    :param f: Path to the input file.
+    :type f: str
+    :param tag: Tag to be used as the sequence. Defaults to None.
+    :type tag: str, optional
+    :param skip: Number of lines to skip (plain text files only). Defaults to None.
+    :type skip: int, optional
+    :param column: Column to parse from the file (plain text files only). Defaults to 0.
+    :type column: int, optional
+    :param separator: Character for column separation (plain text files only). Defaults to "\t".
+    :type separator: str, optional
+    :yields: Sequence from the input file.
+    :rtype: str
+    """
+    import gzip
+
+    if type(f) is not str:
+        src = FASTQ_src(f)  # assume its a stream or file-like object already
+    elif f.endswith(".txt") or f.endswith(".txt.gz"):
+        src = plain_src(f, skip, column, separator)
+    elif f.endswith(".gz"):
+        src = FASTQ_src(gzip.open(f, mode="rt"))
+    elif f.endswith(".bam"):
+        src = BAM_src(f, tag=tag)
+    else:
+        src = FASTQ_src(open(f))
+
+    for _, seq, _ in src:
+        yield seq
+
+
+def read_plain_src_chunked(
+    f: str, tag: str = None, skip: int = None, column: int = 0, separator: str = "\t", chunk_size=100_000_000
+):  
+    """
+    Read data from various sources based on the file type.
+
+    :param f: Path to the input file.
+    :type f: str
+    :param tag: Tag to be used as the sequence. Defaults to None.
+    :type tag: str, optional
+    :param skip: Number of lines to skip (plain text files only). Defaults to None.
+    :type skip: int, optional
+    :param column: Column to parse from the file (plain text files only). Defaults to 0.
+    :type column: int, optional
+    :param separator: Character for column separation (plain text files only). Defaults to "\t".
+    :type separator: str, optional
+    :yields: Sequence from the input file.
+    :rtype: str
+    """
+
+    chunks = pd.read_csv(f, sep=separator, skiprows=skip, chunksize=chunk_size)
+
+    for chunk in chunks:
+        reads = chunk.iloc[:, column]
+        yield reads
+
+
+# TODO: use a partial function to pass args to not use global context
+def find_matches(f: str):
+    """
+    Find matches between query and target sequences.
+
+    :param f: Target file to compare against query sequences.
+    :type f: str
+    :returns: Tuple containing information about the target file, number of barcodes, number of matching barcodes, and the matching ratio.
+    :rtype: tuple
+    """
+    global query_seqs, args
+
+    start = time.time()
+    target = set(
+        read_with_tag(
+            f,
+            args.target_tag,
+            args.target_plain_skip,
+            args.target_plain_column,
+            args.target_separator,
+        )
+    )
+    n_matches = len(target.intersection(query_seqs))
+    pct_matches_target = n_matches / len(target)
+    print(f"queried {f} in {round(time.time()-start, 2)} s")
+
+    return (f, len(target), n_matches, pct_matches_target)
+
+# TODO: implement chunked so it only reads 100M reads every time
+@message_aggregation(logger_name)
+def cmdline():
+    """cmdline."""
+    import argparse
+
+    global args, query_seqs
+
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False,
+        description="calculate sequence intersection between a query and target(s) file(s)",
+    )
+    parser = setup_parser(parser)
+
+    args = parser.parse_args()
+
+    if len(args.target_id) != len(args.target):
+        raise ValueError(
+            f"target_id ({len(args.target_id)}) and target ({len(args.target)}) are different in size"
+        )
+
+    query_seqs = read_with_tag(
+        args.query,
+        args.query_tag,
+        args.query_plain_skip,
+        args.query_plain_column,
+        args.query_separator,
+    )
+    logger.info(f"read query file from '{args.query}'")
+
+    # get unique reads into a global context
+    query_seqs = set(query_seqs)
+    logger.info(f"hashed unique query reads")
+    logger.info(
+        f"querying against {len(args.target)} targets with {args.n_jobs} parallel jobs"
+    )
+
+    with mp.Pool(args.n_jobs) as pool:
+        results = pool.map(find_matches, args.target)
+
+    df = pd.DataFrame(
+        results,
+        columns=["puck_barcode_file", "n_barcodes", "n_matching", "matching_ratio"],
+    )
+    df["puck_barcode_file_id"] = args.target_id
+    df['pass_threshold'] = 0
+    df['pass_threshold'][df['matching_ratio'] > args.min_threshold] = 1
+    df.to_csv(args.summary_output, index=False)
+
+
+if __name__ == "__main__":
+    cmdline()
diff --git a/spacemake/snakemake/scripts/n_intersect_sequences.py b/spacemake/snakemake/scripts/n_intersect_sequences.py
index 485af0ac..e70b0081 100644
--- a/spacemake/snakemake/scripts/n_intersect_sequences.py
+++ b/spacemake/snakemake/scripts/n_intersect_sequences.py
@@ -1,4 +1,5 @@
 import logging
+import os
 
 import multiprocessing as mp
 import pandas as pd
@@ -14,7 +15,8 @@
         --query input_reads.txt \
         --target target_file1.txt target_file2.txt \
         --target-id id1 id2 \
-        --summary-output output_summary.txt
+        --summary-output output_summary.txt \
+        --output target_output1.txt target_output2.txt
 
 Author:
     Daniel León-Periñán
@@ -78,6 +80,14 @@ def setup_parser(parser):
         required=True,
     )
 
+    parser.add_argument(
+        "--output",
+        nargs="+",
+        type=str,
+        help="where to save puck files, in the same order as the --target",
+        default="",
+    )
+
     parser.add_argument(
         "--target-id",
         type=str,
@@ -87,23 +97,9 @@ def setup_parser(parser):
     )
 
     parser.add_argument(
-        "--target-tag",
+        "--target-column",
         type=str,
-        help="if input is a bam file, the specified tag will be used as the sequence",
-        default="CB",
-    )
-
-    parser.add_argument(
-        "--target-plain-skip",
-        type=int,
-        help="number of lines to skip, if target input is a plain text file",
-        default=None,
-    )
-
-    parser.add_argument(
-        "--target-plain-column",
-        type=int,
-        help="column to parse from target, if input is a plain text file",
+        help="column name to parse from target",
         default=None,
     )
 
@@ -119,7 +115,8 @@ def setup_parser(parser):
         "--summary-output",
         type=str,
         help="a summary output file containing the number of matches between query and target, per target file",
-        required=True,
+        required=False,
+        default=""
     )
 
     parser.add_argument(
@@ -130,6 +127,14 @@ def setup_parser(parser):
         required=False,
     )
 
+    parser.add_argument(
+        "--chunksize",
+        type=int,
+        default=int(100*1e6),
+        help="number of query sequences per processing chunk (default: 100M)",
+        required=False,
+    )
+
     parser.add_argument(
         "--n-jobs",
         type=int,
@@ -141,116 +146,54 @@ def setup_parser(parser):
 
     return parser
 
-
-def BAM_src(src: str, tag: str = None) -> tuple:
-    """
-    Generate read data from a BAM file.
-
-    :param src: Path to the BAM file.
-    :type src: str
-    :param tag: Tag to be used as the sequence. Defaults to None.
-    :type tag: str, optional
-    :yields: Tuple containing read name, sequence, and quality.
-    :rtype: tuple
-    """
-
-    import pysam
-
-    bam = pysam.AlignmentFile(src, "rb", check_sq=False)
-    for read in bam.fetch(until_eof=True):
-        if tag is None:
-            yield read.query_name, read.query_sequence, read.query_qualities
-        else:
-            yield None, read.get_tag(tag), None
-
-
-def plain_src(f: str, skip: int = 0, column: int = 0, separator: str = "\t") -> tuple:
-    """
-    Generate read data from a plain text file.
-
-    :param f: Path to the plain text file.
-    :type f: str
-    :param skip: Number of lines to skip. Defaults to 0.
-    :type skip: int, optional
-    :param column: Column to parse from the file. Defaults to 0.
-    :type column: int, optional
-    :param separator: Character for column separation. Defaults to "\t".
-    :type separator: str, optional
-    :yields: Tuple containing read name, sequence, and quality.
-    :rtype: tuple
-    """
-    
-    reads = pd.read_csv(f, sep=separator, skiprows=skip)
-    reads = reads.iloc[:, column]
-
-    for read in reads:
-        yield None, read, None
-
-
-def read_with_tag(
-    f: str, tag: str = None, skip: int = None, column: int = 0, separator: str = "\t"
-) -> str:  
-    """
-    Read data from various sources based on the file type.
-
-    :param f: Path to the input file.
-    :type f: str
-    :param tag: Tag to be used as the sequence. Defaults to None.
-    :type tag: str, optional
-    :param skip: Number of lines to skip (plain text files only). Defaults to None.
-    :type skip: int, optional
-    :param column: Column to parse from the file (plain text files only). Defaults to 0.
-    :type column: int, optional
-    :param separator: Character for column separation (plain text files only). Defaults to "\t".
-    :type separator: str, optional
-    :yields: Sequence from the input file.
-    :rtype: str
-    """
-    import gzip
-
-    if type(f) is not str:
-        src = FASTQ_src(f)  # assume its a stream or file-like object already
-    elif f.endswith(".txt") or f.endswith(".txt.gz"):
-        src = plain_src(f, skip, column, separator)
-    elif f.endswith(".gz"):
-        src = FASTQ_src(gzip.open(f, mode="rt"))
-    elif f.endswith(".bam"):
-        src = BAM_src(f, tag=tag)
-    else:
-        src = FASTQ_src(open(f))
-
-    for _, seq, _ in src:
-        yield seq
-
-
 # TODO: use a partial function to pass args to not use global context
-def find_matches(f: str):
+def find_matches(_target, df=None):
     """
     Find matches between query and target sequences.
 
-    :param f: Target file to compare against query sequences.
-    :type f: str
+    :param i: iterator index for the file argument, to compare against query sequences.
+    :type i: int
     :returns: Tuple containing information about the target file, number of barcodes, number of matching barcodes, and the matching ratio.
     :rtype: tuple
     """
     global query_seqs, args
 
+    if isinstance(_target, int):
+        f_in = args.target[_target]
+        df = pd.read_csv(f_in, sep=args.target_separator)
+        reads = df[args.target_column]
+        target = set(reads)
+        if args.output != "":
+            f_out = args.output[_target]
+    elif isinstance(_target, set):
+        f_in = None
+        target = _target
+        f_out = args.output[0]
+    else:
+        raise ValueError("_target must be either integer (will use as index for 'args.target' or set of unique reads")
+
     start = time.time()
-    target = set(
-        read_with_tag(
-            f,
-            args.target_tag,
-            args.target_plain_skip,
-            args.target_plain_column,
-            args.target_separator,
-        )
-    )
-    n_matches = len(target.intersection(query_seqs))
-    pct_matches_target = n_matches / len(target)
-    print(f"queried {f} in {round(time.time()-start, 2)} s")
+    _intersection = target.intersection(query_seqs)
+    n_matches = len(_intersection)
+    print(f"queried {len(target)} barcodes in {round(time.time()-start, 2)} s")
+
+    if args.output != "":
+        df_matched = df[df[args.target_column].isin(list(_intersection))]
+        df_matched = df_matched[['cell_bc', 'x_pos', 'y_pos']]
+        df_matched.to_csv(f_out, mode='a', header=not os.path.exists(f_out), index=False)
+        print(f"saved puck file into {f_out}")
+
+    return (f_in, len(target), n_matches)
 
-    return (f, len(target), n_matches, pct_matches_target)
+def generate_puck_barcode_summary(df):
+    df_summary = pd.DataFrame({
+        'x_pos_min_px': [df.x_pos.min()],
+        'x_pos_max_px': [df.x_pos.max()],
+        'y_pos_min_px': [df.y_pos.min()],
+        'y_pos_max_px': [df.y_pos.max()]
+    })
 
+    return df_summary
 
 @message_aggregation(logger_name)
 def cmdline():
@@ -271,34 +214,67 @@ def cmdline():
         raise ValueError(
             f"target_id ({len(args.target_id)}) and target ({len(args.target)}) are different in size"
         )
+    
+    if args.output == "" and args.summary_output == "":
+        raise ValueError(
+            "One of --output or --summary-output must have a value"
+        )
 
-    query_seqs = read_with_tag(
-        args.query,
-        args.query_tag,
-        args.query_plain_skip,
-        args.query_plain_column,
-        args.query_separator,
-    )
-    logger.info(f"read query file from '{args.query}'")
-
-    # get unique reads into a global context
-    query_seqs = set(query_seqs)
-    logger.info(f"hashed unique query reads")
-    logger.info(
-        f"querying against {len(args.target)} targets with {args.n_jobs} parallel jobs"
-    )
-
-    with mp.Pool(args.n_jobs) as pool:
-        results = pool.map(find_matches, args.target)
-
-    df = pd.DataFrame(
-        results,
-        columns=["puck_barcode_file", "n_barcodes", "n_matching", "matching_ratio"],
-    )
-    df["puck_barcode_file_id"] = args.target_id
-    df['pass_threshold'] = 0
-    df['pass_threshold'][df['matching_ratio'] > args.min_threshold] = 1
-    df.to_csv(args.summary_output, index=False)
+    # Create a df for the final results
+    result_df = pd.DataFrame()
+
+    # Load the query file as chunks
+    query_df = pd.read_csv(args.query, sep=args.query_separator, skiprows=args.query_plain_skip, chunksize=args.chunksize)
+
+    if len(args.output) == 1:
+        df = pd.read_csv(args.target[0], sep=args.target_separator)
+        reads = df[args.target_column]
+        target = set(reads)
+
+    # Process in chunks
+    for query_df_chunk in query_df:
+        query_seqs = query_df_chunk.iloc[:, args.query_plain_column]
+        logger.info(f"read chunk of {round(len(query_seqs)/1e6, 2)}M sequences from query file at '{args.query}'")
+
+        # get unique reads into a global context
+        query_seqs = set(query_seqs)
+        logger.info(f"hashed unique query reads")
+
+        if len(args.target) == 1:
+            _, n_barcodes, n_matches = find_matches(target, df)
+            results = [(args.target[0], n_barcodes, n_matches)]
+        elif len(args.target) > 1:
+            logger.info(
+                f"querying against {len(args.target)} targets with {args.n_jobs} parallel jobs"
+            )
+            with mp.Pool(args.n_jobs) as pool:
+                results = pool.map(find_matches, range(len(args.target)))
+
+        result_df_chunk = pd.DataFrame(
+            results,
+            columns=["puck_barcode_file", "n_barcodes", "n_matching"],
+        )
+        result_df_chunk["puck_barcode_file_id"] = args.target_id
+        result_df = pd.concat([result_df, result_df_chunk])
+    
+    if len(args.output) == 1 and args.summary_output != "":
+        df_summary = generate_puck_barcode_summary(df)
+        result_df = result_df.groupby(['puck_barcode_file', 'puck_barcode_file_id', 'n_barcodes']).sum().reset_index()
+        df_summary['puck_barcode_file'] = result_df['puck_barcode_file']
+        df_summary['puck_barcode_file_id'] = result_df['puck_barcode_file_id']
+        df_summary['n_barcodes'] = result_df['n_barcodes']
+        df_summary['n_matching'] = result_df['n_matching']
+        df_summary['matching_ratio'] = df_summary['n_matching']/df_summary['n_barcodes']
+        df_summary['parsed_barcode_file'] = args.output
+        df_summary.to_csv(args.summary_output, index=False)
+    
+    elif args.summary_output != "":
+        # group per puck_barcode_file, and compute sum
+        result_df = result_df.groupby(['puck_barcode_file', 'puck_barcode_file_id', 'n_barcodes']).sum().reset_index()
+        result_df['matching_ratio'] = result_df['n_matching']/result_df['n_barcodes']
+        result_df['pass_threshold'] = 0
+        result_df['pass_threshold'][result_df['matching_ratio'] > args.min_threshold] = 1
+        result_df.to_csv(args.summary_output, index=False)
 
 
 if __name__ == "__main__":
diff --git a/spacemake/snakemake/scripts/snakemake_helper_functions.py b/spacemake/snakemake/scripts/snakemake_helper_functions.py
index 3d4285d7..4c874b2b 100644
--- a/spacemake/snakemake/scripts/snakemake_helper_functions.py
+++ b/spacemake/snakemake/scripts/snakemake_helper_functions.py
@@ -319,7 +319,7 @@ def parse_barcode_flavors(
         cell_raw="None",
         score_threshold=0.0,
         min_opseq_score=22,
-        bam_tags="CR:{cell},MI:{UMI}",
+        bam_tags="CR:{cell},CB:{cell},MI:{UMI}",
     ),
 ):
     """
@@ -910,12 +910,12 @@ def get_qc_sheet_input_files(wildcards):
     return to_return
 
 
-def get_bam_tag_names(project_id, sample_id):
+def get_bam_tag_names(project_id, sample_id, default_tags="CR:{cell},CB:{cell},MI:{UMI},RG:{assigned}"):
     barcode_flavor = project_df.get_metadata(
         "barcode_flavor", project_id=project_id, sample_id=sample_id
     )
 
-    bam_tags = config["barcode_flavors"][barcode_flavor]["bam_tags"]
+    bam_tags = config["barcode_flavors"][barcode_flavor].get("bam_tags", default_tags)
 
     tag_names = {}
 
@@ -938,6 +938,13 @@ def get_puck_file(wildcards):
         return []
     else:
         return {"barcode_file": puck_barcode_file}
+    
+def get_all_puck_files(wildcards):
+    _, pbf = project_df.get_puck_barcode_ids_and_files(
+        project_id=wildcards.project_id, sample_id=wildcards.sample_id
+    )
+
+    return {"barcode_file": pbf}
 
 
 def get_puck_collection_stitching_input(wildcards, to_mesh=False):
@@ -984,6 +991,26 @@ def get_barcode_files_matching_summary_input(wildcards):
     }
 
 
+def get_barcode_summary_files_matching_summary_input(wildcards):
+    pbf_ids, _ = project_df.get_puck_barcode_ids_and_files(
+        project_id=wildcards.project_id, sample_id=wildcards.sample_id
+    )
+
+    parsed_spatial_barcode_summary_files = [
+        expand(
+            parsed_spatial_barcodes_summary,
+            project_id=wildcards.project_id,
+            sample_id=wildcards.sample_id,
+            puck_barcode_file_id=pbf_id,
+        )[0]
+        for pbf_id in pbf_ids
+    ]
+
+    return {
+        "matched_barcode_files_summary": parsed_spatial_barcode_summary_files
+    }
+
+
 def get_barcode_files(wildcards):
     pbf_ids, pbfs = project_df.get_puck_barcode_ids_and_files(
         project_id=wildcards.project_id, sample_id=wildcards.sample_id
diff --git a/spacemake/snakemake/variables.py b/spacemake/snakemake/variables.py
index 7b6a4fb4..3b854d09 100644
--- a/spacemake/snakemake/variables.py
+++ b/spacemake/snakemake/variables.py
@@ -1,5 +1,7 @@
 # set root dir where the processed_data goes
 project_dir = "projects/{project_id}"
+config_path = "config.yaml"
+project_df = "project_df.csv"
 
 #################
 # DIRECTORY STR #
@@ -17,6 +19,10 @@
 downsampled_data_prefix = illumina_root + "/downsampled_data"
 downsampled_data_root = downsampled_data_prefix + "{downsampling_percentage}"
 
+log_dir = complete_data_root + '/logs'
+stats_dir = complete_data_root + '/stats'
+plots_dir = complete_data_root + '/plots'
+
 ##############
 # Demux vars #
 ##############
@@ -75,7 +81,9 @@
 reads_type_out = split_reads_read_type
 barcode_readcounts_suffix = "{polyA_adapter_trimmed}.txt.gz"
 barcode_readcounts = complete_data_root + "/out_readcounts" + barcode_readcounts_suffix
+barcode_readcounts_log = barcode_readcounts + ".log"
 barcode_readcounts_prealigned = complete_data_root + "/out_readcounts_prealigned.txt.gz"
+barcode_readcounts_prealigned_log = barcode_readcounts_prealigned + ".log"
 strand_info = split_reads_strand_type
 
 # united final.bam
@@ -91,6 +99,10 @@
     complete_data_root
     + "/puck_barcode_files/spatial_barcodes_{puck_barcode_file_id}.csv"
 )
+parsed_spatial_barcodes_summary = (
+    complete_data_root
+    + "/puck_barcode_files/spatial_barcodes_summary_{puck_barcode_file_id}.csv"
+)
 parsed_spatial_barcodes_pc = (
     complete_data_root
     + "/puck_barcode_files/spatial_barcodes_puck_collection.csv"
@@ -243,6 +255,7 @@
 #  dropseq rules and vars #
 # #########################
 tagged_bam = complete_data_root + "/unaligned_bc_tagged.bam"
+tagged_bam_log = tagged_bam + ".log"
 unassigned = complete_data_root + "/unaligned_bc_unassigned.bam"
 
 # trim smart adapter from the reads
diff --git a/spacemake/spatial/__init__.py b/spacemake/spatial/__init__.py
index 251a9fbe..ebc4726a 100644
--- a/spacemake/spatial/__init__.py
+++ b/spacemake/spatial/__init__.py
@@ -1,6 +1,6 @@
-# include in top level for backward compatibility
-from .util import compute_neighbors, compute_islands, detect_tissue, \
-    create_mesh, create_meshed_adata
-# added novosparc_reconstruction for backward compatibility
-from . import novosparc_integration as novosparc_reconstruction
-from . import puck_collection as puck_collection
+# # include in top level for backward compatibility
+# from .util import compute_neighbors, compute_islands, detect_tissue, \
+#     create_mesh, create_meshed_adata
+# # added novosparc_reconstruction for backward compatibility
+# from . import novosparc_integration as novosparc_reconstruction
+# from . import puck_collection as puck_collection
diff --git a/spacemake/spatial/cmdline.py b/spacemake/spatial/cmdline.py
index 257d2695..9971507e 100644
--- a/spacemake/spatial/cmdline.py
+++ b/spacemake/spatial/cmdline.py
@@ -1,8 +1,5 @@
-from ..config import ConfigFile
-from ..project_df import ProjectDF
-
-from .he_integration import create_aggregated_expression_img, \
-    create_spot_expression_img
+#from ..config import ConfigFile
+#from ..project_df import ProjectDF
 from ..util import message_aggregation, bool_in_str, str2bool
 from ..errors import SpacemakeError
 
@@ -42,7 +39,7 @@ def get_expression_img_parser(with_umi_cutoff = False):
 
     return parser
 
-def setup_spatial_parser(spmk, parent_parser_subparsers):
+def setup_spatial_parser(parent_parser_subparsers):
     parser = parent_parser_subparsers.add_parser('spatial',
         help = 'spacemake spatial commands')
 
@@ -53,7 +50,7 @@ def setup_spatial_parser(spmk, parent_parser_subparsers):
         parents=[get_expression_img_parser()])
 
     aggregated_img_parser.set_defaults(
-        func=lambda args: create_expression_img_cmdline(spmk, args,
+        func=lambda args: create_expression_img_cmdline(args,
             'aggregated'))
 
     spot_img_parser = subparsers.add_parser(
@@ -61,14 +58,15 @@ def setup_spatial_parser(spmk, parent_parser_subparsers):
         parents=[get_expression_img_parser()])
 
     spot_img_parser.set_defaults(
-        func=lambda args: create_expression_img_cmdline(spmk, args,
+        func=lambda args: create_expression_img_cmdline(args,
             'spot'))
 
 @message_aggregation(logger_name)
-def create_expression_img_cmdline(spmk, args, img_type):
+def create_expression_img_cmdline(args, img_type):
     import cv2 
-
     logger.info('Loading dge file...')
+    from spacemake.smk import Spacemake
+    spmk = Spacemake()
 
     if str2bool(args['processed_data']):
         if not 'umi_cutoff' in args:
@@ -89,9 +87,11 @@ def create_expression_img_cmdline(spmk, args, img_type):
 
     logger.info(f'Generating {img_type} expression image...')
     if img_type == 'spot':
+        from .he_integration import create_spot_expression_img
         img, img_bw = create_spot_expression_img(adata,
             binary=str2bool(args['binary']))
     elif img_type == 'aggregated':
+        from .he_integration import create_aggregated_expression_img
         img, img_bw = create_aggregated_expression_img(
             adata,
             binary_top_qth_percentile=int(args['binary_top_qth_percentile']))
@@ -99,4 +99,4 @@ def create_expression_img_cmdline(spmk, args, img_type):
     if str2bool(args['binary']):
         img = img_bw
 
-    cv2.imwrite(args['out_img'], img)
+    cv2.imwrite(args['out_img'], img)
\ No newline at end of file
diff --git a/spacemake/spatial/he_integration.py b/spacemake/spatial/he_integration.py
index ed1f831b..b84f0dbe 100644
--- a/spacemake/spatial/he_integration.py
+++ b/spacemake/spatial/he_integration.py
@@ -11,20 +11,21 @@
 logger_name = "spacemake.spatial"
 logger = logging.getLogger(logger_name)
 
+
 def create_spot_expression_img(
-        adata: anndata.AnnData,
-        binary : bool=True,
-    ) -> (numpy.ndarray, numpy.ndarray):
+    adata: anndata.AnnData,
+    binary: bool = True,
+) -> (numpy.ndarray, numpy.ndarray):
     """
     Creates a binary/grayscale image based on spatial expression data.
 
     Given a spacemake processed AnnData object this function will create
-    two images based on the spatial expression values, one grayscale and 
-    one binary (black and white). For each observation in the data 
+    two images based on the spatial expression values, one grayscale and
+    one binary (black and white). For each observation in the data
     (spot/spatial-unit) a circle will be drawn, where the diameter of the
-    circle is equal to the spot_diameter_um  provided in the puck parameters 
-    for this sample. In the binary image each circle will be black and 
-    for the grayscale image the color of each circle will correspond to the 
+    circle is equal to the spot_diameter_um  provided in the puck parameters
+    for this sample. In the binary image each circle will be black and
+    for the grayscale image the color of each circle will correspond to the
     UMI count of that spot. Both image returned are 8-bit.
 
     :param adata: A spacemake processed AnnData object.
@@ -35,73 +36,81 @@ def create_spot_expression_img(
     df = adata.obs.copy()
     x_pos_max = df.x_pos.max()
     y_pos_max = df.y_pos.max()
-    coord_by_um = adata.uns['puck_variables']['coord_by_um']
+    coord_by_um = adata.uns["puck_variables"]["coord_by_um"]
 
     width = int(x_pos_max / coord_by_um)
     height = int(y_pos_max / coord_by_um)
 
-    spot_diameter = int(adata.uns['puck_variables']['spot_diameter_um'])
+    spot_diameter = int(adata.uns["puck_variables"]["spot_diameter_um"])
 
     width = width + spot_diameter
     height = height + spot_diameter
-    
+
     spot_img = np.zeros((height, width), np.uint8)
     spot_img_bw = np.zeros((height, width), np.uint8)
 
     df.total_counts = df.total_counts * 255 / df.total_counts.max()
-    
-    df.x_pos = df.x_pos * height / x_pos_max + int(spot_diameter/2)
-    df.y_pos = df.y_pos * width / y_pos_max + int(spot_diameter/2)
-    
+
+    df.x_pos = df.x_pos * height / x_pos_max + int(spot_diameter / 2)
+    df.y_pos = df.y_pos * width / y_pos_max + int(spot_diameter / 2)
+
     for i, row in df.iterrows():
-        color = int(row['total_counts'])
+        color = int(row["total_counts"])
 
         # draw the circle
-        cv2.circle(spot_img,
-                   (int(row['y_pos']), int(row['x_pos'])),
-                   int(spot_diameter/2),
-                   (color, color, color), -1)
-
+        cv2.circle(
+            spot_img,
+            (int(row["y_pos"]), int(row["x_pos"])),
+            int(spot_diameter / 2),
+            (color, color, color),
+            -1,
+        )
 
         # draw the circle
-        cv2.circle(spot_img_bw,
-                   (int(row['y_pos']), int(row['x_pos'])),
-                   int(spot_diameter/2),
-                   (255, 255, 255), -1)
+        cv2.circle(
+            spot_img_bw,
+            (int(row["y_pos"]), int(row["x_pos"])),
+            int(spot_diameter / 2),
+            (255, 255, 255),
+            -1,
+        )
 
-        
     spot_img = 255 - spot_img
     spot_img_bw = 255 - spot_img_bw
-    
+
     spot_img = cv2.resize(spot_img, (1000, 1000), cv2.INTER_AREA)
     spot_img_bw = cv2.resize(spot_img_bw, (1000, 1000), cv2.INTER_AREA)
-    
-    spot_img_bw[spot_img_bw<255] = 0
-    
+
+    spot_img_bw[spot_img_bw < 255] = 0
+
     spot_img_cnt = np.where(spot_img_bw < 255)
-    spot_img = spot_img[spot_img_cnt[0].min():spot_img_cnt[0].max(),
-                        spot_img_cnt[1].min():spot_img_cnt[1].max()]
+    spot_img = spot_img[
+        spot_img_cnt[0].min() : spot_img_cnt[0].max(),
+        spot_img_cnt[1].min() : spot_img_cnt[1].max(),
+    ]
+
+    spot_img_bw = spot_img_bw[
+        spot_img_cnt[0].min() : spot_img_cnt[0].max(),
+        spot_img_cnt[1].min() : spot_img_cnt[1].max(),
+    ]
 
-    spot_img_bw = spot_img_bw[spot_img_cnt[0].min():spot_img_cnt[0].max(),
-                              spot_img_cnt[1].min():spot_img_cnt[1].max()]
-    
     return spot_img, spot_img_bw
 
+
 def create_aggregated_expression_img(
-        adata : anndata.AnnData,
-        binary_top_qth_percentile : int =30
-    ) -> (numpy.ndarray, numpy.ndarray):
+    adata: anndata.AnnData, binary_top_qth_percentile: int = 30
+) -> (numpy.ndarray, numpy.ndarray):
     """
     Creates an aggregated binary/grayscale image based on spatial expression data.
 
-    Similar to create_spot_expression_img. Rather than draw a circle for each 
-    spot, data will be aggregated into a 500 x 500 pixel image. For grayscale 
-    images the value of each pixel will be corresponding to the sum of UMI 
-    counts under that pixel. For the binary image, pixels will be filtered 
-    based on the binary_top_qth_percentile parameter. After aggregation only 
-    the binary_top_qth_percentile pixels (the top 30% of pixels based on 
-    expression value) will be set. After this step the image will be smoothed 
-    and pixels with enough set neighbors will also be set. Both images 
+    Similar to create_spot_expression_img. Rather than draw a circle for each
+    spot, data will be aggregated into a 500 x 500 pixel image. For grayscale
+    images the value of each pixel will be corresponding to the sum of UMI
+    counts under that pixel. For the binary image, pixels will be filtered
+    based on the binary_top_qth_percentile parameter. After aggregation only
+    the binary_top_qth_percentile pixels (the top 30% of pixels based on
+    expression value) will be set. After this step the image will be smoothed
+    and pixels with enough set neighbors will also be set. Both images
     returned are 8-bit.
 
 
@@ -114,42 +123,48 @@ def create_aggregated_expression_img(
     """
     df = adata.obs.copy()
     # load the clusters in their places as per coordinates
-    df.x_pos = df.x_pos.astype('int')
-    df.y_pos = df.y_pos.astype('int')
+    df.x_pos = df.x_pos.astype("int")
+    df.y_pos = df.y_pos.astype("int")
 
     max_x = df.x_pos.max()
     max_y = df.y_pos.max()
 
-    img = np.zeros((max_x+1, max_y+1), np.uint8)
+    img = np.zeros((max_x + 1, max_y + 1), np.uint8)
 
     img[df.x_pos, df.y_pos] = df.total_counts
-    
+
     if max_x <= 500 or max_y <= 500:
-        raise SpacemakeError('Underlying spatial coordinates cannot be aggregated\n'+
-            'Only coordinates with maximum values for x_pos and y_pos ' +
-            'both higher than 500 can be aggregated\n' +
-            'Use `spacemake spatial create_spot_expression_img` instead')
+        raise SpacemakeError(
+            "Underlying spatial coordinates cannot be aggregated\n"
+            + "Only coordinates with maximum values for x_pos and y_pos "
+            + "both higher than 500 can be aggregated\n"
+            + "Use `spacemake spatial create_spot_expression_img` instead"
+        )
 
-    scale_f = int(img.shape[0]/500)
+    scale_f = int(img.shape[0] / 500)
 
     # creating a 500x500 pixel image
-    img_scaled=np.add.reduceat(img, range(0, img.shape[0],scale_f))
-    img_scaled=np.add.reduceat(img_scaled, range(0, img.shape[1],scale_f), axis=1)
+    img_scaled = np.add.reduceat(img, range(0, img.shape[0], scale_f))
+    img_scaled = np.add.reduceat(img_scaled, range(0, img.shape[1], scale_f), axis=1)
 
     # get only top 30%, by default
     if binary_top_qth_percentile > 0 and binary_top_qth_percentile <= 100:
-        filter_val = np.percentile(img_scaled.flatten(), 100 - binary_top_qth_percentile)
+        filter_val = np.percentile(
+            img_scaled.flatten(), 100 - binary_top_qth_percentile
+        )
     else:
-        raise ValueError(f'binary_top_qth_percentile has to be between 1 and 100.'
-            f'The value provided ({binary_top_qth_percentile}) is invalid')
-    
+        raise ValueError(
+            f"binary_top_qth_percentile has to be between 1 and 100."
+            f"The value provided ({binary_top_qth_percentile}) is invalid"
+        )
+
     img_scaled_bw = img_scaled.copy()
-    
+
     img_scaled_bw[img_scaled_bw > filter_val] = 255
-    img_scaled_bw[img_scaled_bw <=filter_val] = 0
+    img_scaled_bw[img_scaled_bw <= filter_val] = 0
+
+    img_scaled_bw = img_scaled_bw.astype(np.uint8)
 
-    img_scaled_bw=img_scaled_bw.astype(np.uint8)
-    
     # fill black
     img_scaled_bw = img_scaled_bw | fill_image(img_scaled_bw, 15, 5)
     # fill white
@@ -160,9 +175,10 @@ def create_aggregated_expression_img(
 
     img_scaled = img_scaled * 255 / img_scaled.max()
     img_scaled = img_scaled.astype(np.uint8)
-    
+
     return ~img_scaled, ~img_scaled_bw
 
+
 def fill_image(X, n_neighbors=3, infer_square_side=3):
     """fill_image.
 
@@ -177,18 +193,21 @@ def fill_image(X, n_neighbors=3, infer_square_side=3):
 
     for i in range(infer_square_side):
         for j in range(infer_square_side):
-            Y[i:(width+i), j:(height+j)] = Y[i:(width+i), j:(height+j)] + X
-            
+            Y[i : (width + i), j : (height + j)] = (
+                Y[i : (width + i), j : (height + j)] + X
+            )
+
     # get the middle
-    Y = Y[1:(width+1), 1:(height+1)] - X
-    
+    Y = Y[1 : (width + 1), 1 : (height + 1)] - X
+
     # get only the pixels which have at least n_neighbors black neighbors and set them black
-    Y[np.where(Y < 255*n_neighbors)] = 0
-    Y[np.where(Y >= 255*n_neighbors)] = 255
+    Y[np.where(Y < 255 * n_neighbors)] = 0
+    Y[np.where(Y >= 255 * n_neighbors)] = 255
     Y = Y.astype(np.uint8)
 
     return Y
 
+
 def fill_holes_by_neighbors(dat, dist=2):
     """fill_holes_by_neighbors.
 
@@ -199,16 +218,17 @@ def fill_holes_by_neighbors(dat, dist=2):
     bottom = np.zeros(dat.shape, dtype=np.uint8)
     left = np.zeros(dat.shape, dtype=np.uint8)
     right = np.zeros(dat.shape, dtype=np.uint8)
-    h,w = dat.shape
-    
+    h, w = dat.shape
+
     for i in range(dist):
-        top[0:(h-i), :] = top[0:(h-i), :] | dat[i:h, :]
-        bottom[i:h, :] = bottom[i:h, :] | dat[0:(h-i), :]
-        right[:, 0:(w-i)] = right[:, 0:(w-i)] | dat[:, i:w]
-        left[:, i:w] = right[:, i:w] | dat[:, 0:(w-i)]
-    
+        top[0 : (h - i), :] = top[0 : (h - i), :] | dat[i:h, :]
+        bottom[i:h, :] = bottom[i:h, :] | dat[0 : (h - i), :]
+        right[:, 0 : (w - i)] = right[:, 0 : (w - i)] | dat[:, i:w]
+        left[:, i:w] = right[:, i:w] | dat[:, 0 : (w - i)]
+
     return top & bottom & left & right
 
+
 def load_he_img(he_path, bw_threshold=None):
     """load_he_img.
 
@@ -220,12 +240,15 @@ def load_he_img(he_path, bw_threshold=None):
 
     # create binary image
     if bw_threshold is None:
-        thresh, he_bw = cv2.threshold(he_gray, 127, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
+        thresh, he_bw = cv2.threshold(
+            he_gray, 127, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU
+        )
     else:
         he_bw = cv2.threshold(he_gray, bw_threshold, 255, cv2.THRESH_BINARY)[1]
-        
+
     return he, he_gray, he_bw
 
+
 def align_he_img(he_path, expression_img, bw_threshold=None, use_bw=True):
     """align_he_img.
 
@@ -235,133 +258,139 @@ def align_he_img(he_path, expression_img, bw_threshold=None, use_bw=True):
     :param use_bw:
     """
     he, he_gray, he_bw = load_he_img(he_path, bw_threshold=bw_threshold)
-    
+
     he_orig = he.copy()
-    
+
     # find the scale by which we resize the images
     expression_img_height, expression_img_width = expression_img.shape
     he_height, he_width = he_gray.shape
-    
+
     max_zoom = 3
-    
-    height_ratio = he_height / (max_zoom*expression_img_height)
-    width_ratio = he_width / (max_zoom*expression_img_width)
-    
+
+    height_ratio = he_height / (max_zoom * expression_img_height)
+    width_ratio = he_width / (max_zoom * expression_img_width)
+
     resize_type = cv2.INTER_NEAREST if use_bw else cv2.INTER_AREA
-    
+
     if height_ratio > 1.0 and width_ratio > 1.0:
         # we scale the he image
         if height_ratio > width_ratio:
-            scale_f = 1.0/width_ratio
+            scale_f = 1.0 / width_ratio
         else:
-            scale_f = 1.0/height_ratio
-        
-        dim = (int(he_width*scale_f)+1,
-               int(he_height*scale_f)+1)
-        
-        he = cv2.resize(he, dim, 0,0,cv2.INTER_AREA)
-        he_gray = cv2.resize(he_gray, dim, 0,0,cv2.INTER_AREA)
-        he_bw = cv2.resize(he_bw, dim, 0,0,cv2.INTER_NEAREST)
-        
+            scale_f = 1.0 / height_ratio
+
+        dim = (int(he_width * scale_f) + 1, int(he_height * scale_f) + 1)
+
+        he = cv2.resize(he, dim, 0, 0, cv2.INTER_AREA)
+        he_gray = cv2.resize(he_gray, dim, 0, 0, cv2.INTER_AREA)
+        he_bw = cv2.resize(he_bw, dim, 0, 0, cv2.INTER_NEAREST)
+
     else:
         if height_ratio > width_ratio:
             scale_f = width_ratio
         else:
             scale_f = height_ratio
-        
-        expression_img = cv2.resize(expression_img,
-                                      (int(expression_img_width*scale_f),
-                                       int(expression_img_height*scale_f)),
-                                      resize_type)
-    
+
+        expression_img = cv2.resize(
+            expression_img,
+            (int(expression_img_width * scale_f), int(expression_img_height * scale_f)),
+            resize_type,
+        )
+
     he_bw = ~he_bw
-    
+
     for i in range(10):
-        he_bw = fill_holes_by_neighbors(he_bw, int(max(he_bw.shape)/200))
-    
+        he_bw = fill_holes_by_neighbors(he_bw, int(max(he_bw.shape) / 200))
+
     he_bw = ~he_bw
-    
+
     highest_cor = 0.0
     scale_dim = None
     align_res = None
     rotate = 0
     flip = False
-    
-    he_res =None
-        
+
+    he_res = None
+
     for make_flip in [False, True]:
         he_copy = he_bw.copy() if use_bw else he_gray.copy()
-        
+
         if make_flip:
             he_copy = cv2.flip(he_copy, 0)
-        
-        for rotate_n in [0,1,2,3]:
+
+        for rotate_n in [0, 1, 2, 3]:
             he_rotated = he_copy
             for r in range(rotate_n):
-                he_rotated = cv2.rotate(he_rotated, cv2.ROTATE_90_CLOCKWISE).copy()    
-            
-            for scale_x in np.linspace(1.0, 1.0/max_zoom + 0.01, 50):
-                scales_y = (min(1.0, scale_x*1.1), max(1.0/max_zoom + 0.01, scale_x*0.9))
+                he_rotated = cv2.rotate(he_rotated, cv2.ROTATE_90_CLOCKWISE).copy()
+
+            for scale_x in np.linspace(1.0, 1.0 / max_zoom + 0.01, 50):
+                scales_y = (
+                    min(1.0, scale_x * 1.1),
+                    max(1.0 / max_zoom + 0.01, scale_x * 0.9),
+                )
                 for scale_y in np.linspace(scales_y[0], scales_y[1], 10):
-                    dim = (int(he_bw.shape[1] * scale_x),
-                           int(he_bw.shape[0] * scale_y))
-                    
-                    he_scaled = cv2.resize(he_rotated, dim, 0,0, resize_type)
-                    m_res = cv2.matchTemplate(he_scaled, expression_img, cv2.TM_CCOEFF_NORMED)
-                    
+                    dim = (int(he_bw.shape[1] * scale_x), int(he_bw.shape[0] * scale_y))
+
+                    he_scaled = cv2.resize(he_rotated, dim, 0, 0, resize_type)
+                    m_res = cv2.matchTemplate(
+                        he_scaled, expression_img, cv2.TM_CCOEFF_NORMED
+                    )
+
                     if m_res.max() > highest_cor:
                         highest_cor = m_res.max()
                         scale_dim = dim
                         align_res = m_res
                         rotate = rotate_n
                         flip = make_flip
-    
+
     # find the boundaries of the align
     min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(align_res)
     top_left = max_loc
-    bottom_right = (top_left[0] + expression_img.shape[1],
-                    top_left[1] + expression_img.shape[0])
-    
-    #expression_img[expression_img > 0] = 255
+    bottom_right = (
+        top_left[0] + expression_img.shape[1],
+        top_left[1] + expression_img.shape[0],
+    )
+
+    # expression_img[expression_img > 0] = 255
     expression_img_clr = cv2.cvtColor(expression_img, cv2.COLOR_GRAY2BGR)
-    
+
     if flip:
         he = cv2.flip(he, 0)
         he_gray = cv2.flip(he_gray, 0)
         he_bw = cv2.flip(he_bw, 0)
         he_orig = cv2.flip(he_orig, 0)
-    
+
     for r in range(rotate):
         he = cv2.rotate(he, cv2.ROTATE_90_CLOCKWISE)
         he_gray = cv2.rotate(he_gray, cv2.ROTATE_90_CLOCKWISE)
         he_bw = cv2.rotate(he_bw, cv2.ROTATE_90_CLOCKWISE)
         he_orig = cv2.rotate(he_orig, cv2.ROTATE_90_CLOCKWISE)
-    
-    he = cv2.resize(he, scale_dim, 0,0,cv2.INTER_AREA)
-    he_bw_res = cv2.resize(he_bw, scale_dim, 0,0, cv2.INTER_NEAREST)
-    he_gray_res = cv2.resize(he_gray, scale_dim, 0,0, cv2.INTER_AREA)
+
+    he = cv2.resize(he, scale_dim, 0, 0, cv2.INTER_AREA)
+    he_bw_res = cv2.resize(he_bw, scale_dim, 0, 0, cv2.INTER_NEAREST)
+    he_gray_res = cv2.resize(he_gray, scale_dim, 0, 0, cv2.INTER_AREA)
 
     return highest_cor, he, he_orig, top_left, bottom_right
 
+
 def align_he_spot_img(
-        adata : anndata.AnnData,
-        he_path : str
-    ) -> (numpy.ndarray, numpy.ndarray):
+    adata: anndata.AnnData, he_path: str
+) -> (numpy.ndarray, numpy.ndarray):
     """
     Align a H&E image with a spacemake processed data, based on spot expression.
 
-    Takes an H&E image and a spacemake processed dataset, and alignes the two, 
-    trying to find the most probably position of the spatial dataset within the 
-    H&E image. This function should be used for datasets with a spot_diameter_um 
+    Takes an H&E image and a spacemake processed dataset, and alignes the two,
+    trying to find the most probably position of the spatial dataset within the
+    H&E image. This function should be used for datasets with a spot_diameter_um
     higher than or equal to 10 microns, such as Visium or Slide-seq.
 
     :param adata: A spacemake processed AnnData object.
     :type adata: anndata.AnnData
     :param he_path: Path to the H&E image to be aligned.
     :type he_path: str
-    :returns: A tuple of (original_he, aligned_he). original_he will contain the 
-        original H&E data together with the aligned spots and a blue rectangle 
-        showing the aligned region. aligned_he will contain the part of the 
+    :returns: A tuple of (original_he, aligned_he). original_he will contain the
+        original H&E data together with the aligned spots and a blue rectangle
+        showing the aligned region. aligned_he will contain the part of the
         H&E image aligned.
     :rtype: (numpy.ndarray, numpy.ndarray)
     """
@@ -372,54 +401,58 @@ def align_he_spot_img(
 
     # find the contour of the spot image
     spot_img_cnt = np.where(spot_img < 255)
-    spot_img = spot_img[spot_img_cnt[0].min():spot_img_cnt[0].max(),
-                        spot_img_cnt[1].min():spot_img_cnt[1].max()]
+    spot_img = spot_img[
+        spot_img_cnt[0].min() : spot_img_cnt[0].max(),
+        spot_img_cnt[1].min() : spot_img_cnt[1].max(),
+    ]
 
     highest_cor, he_res, he_orig, tl, br = align_he_img(
-        he_path,
-        expression_img = spot_img,
-        use_bw = True)
+        he_path, expression_img=spot_img, use_bw=True
+    )
 
     he_res_ratio = he_res.shape[1] / he_res.shape[0]
 
-    he_orig = cv2.resize(he_orig,
-                     (int(he_orig.shape[0] * he_res_ratio),he_orig.shape[0]),
-                     cv2.INTER_AREA)
+    he_orig = cv2.resize(
+        he_orig,
+        (int(he_orig.shape[0] * he_res_ratio), he_orig.shape[0]),
+        cv2.INTER_AREA,
+    )
 
     horizontal_ratio = he_orig.shape[0] / he_res.shape[0]
     vertical_ratio = he_orig.shape[1] / he_res.shape[1]
-    
+
     box_tl = int(tl[0] * horizontal_ratio), int(tl[1] * vertical_ratio)
     box_br = int(br[0] * horizontal_ratio), int(br[1] * vertical_ratio)
 
-    spot_img_resized = cv2.resize(spot_img,
-                                  (box_br[0] - box_tl[0], box_br[1] - box_tl[1]),
-                                   cv2.INTER_NEAREST)
+    spot_img_resized = cv2.resize(
+        spot_img, (box_br[0] - box_tl[0], box_br[1] - box_tl[1]), cv2.INTER_NEAREST
+    )
 
     align_bw = cv2.cvtColor(spot_img_resized, cv2.COLOR_GRAY2BGR)
 
-    he_align = he_orig[box_tl[1]:box_br[1], box_tl[0]:box_br[0]].copy()
-    he_orig[box_tl[1]:box_br[1], box_tl[0]:box_br[0]] &= align_bw
+    he_align = he_orig[box_tl[1] : box_br[1], box_tl[0] : box_br[0]].copy()
+    he_orig[box_tl[1] : box_br[1], box_tl[0] : box_br[0]] &= align_bw
+
+    cv2.rectangle(he_orig, box_tl, box_br, (180, 233, 86), 3)
 
-    cv2.rectangle(he_orig, box_tl, box_br,  (180, 233, 86), 3)
-    
     return he_orig, he_align
 
+
 def align_he_aggregated_img(
-        adata : anndata.AnnData,
-        he_path : str,
-        bw_threshold : int=200,
-        binary_top_qth_percentile : int=30,
-        box_size : float=0.5
-    ) -> (numpy.ndarray, numpy.ndarray):
+    adata: anndata.AnnData,
+    he_path: str,
+    bw_threshold: int = 200,
+    binary_top_qth_percentile: int = 30,
+    box_size: float = 0.5,
+) -> (numpy.ndarray, numpy.ndarray):
     """align_he_aggregated_img.
-    Align a H&E image with a spacemake processed data, based on aggregated 
+    Align a H&E image with a spacemake processed data, based on aggregated
     expression.
 
-    Takes an H&E image and a spacemake processed dataset, and alignes the two, 
-    trying to find the most probably position of the spatial dataset within the 
-    H&E image. First the spatial data will be aggregated in a 500 x 500 pixel 
-    image, then the best align between the spatial expression image and the H&E 
+    Takes an H&E image and a spacemake processed dataset, and alignes the two,
+    trying to find the most probably position of the spatial dataset within the
+    H&E image. First the spatial data will be aggregated in a 500 x 500 pixel
+    image, then the best align between the spatial expression image and the H&E
     will be found. This function should be used for datasets with a
     spot_diameter_um smaller than 10 microns, such as Visium or Slide-seq.
 
@@ -427,7 +460,7 @@ def align_he_aggregated_img(
     :type adata: anndata.AnnData
     :param he_path: Path to the H&E image to be aligned.
     :type he_path: str
-    :param bw_threshold: 
+    :param bw_threshold:
     :type bw_threshold: int
     :param binary_top_qth_percentile:
     :type binary_top_qth_percentile: int
@@ -435,8 +468,8 @@ def align_he_aggregated_img(
         only the middle part (a box with 50% x 50% pixels, wrt the original size).
         This parameter controls the size of the aligning box.
     :type box_size: float
-    :returns: A tuple of (original_he, aligned_he). original_he will contain the 
-        original H&E data together with the aggregated image overlayed, together 
+    :returns: A tuple of (original_he, aligned_he). original_he will contain the
+        original H&E data together with the aggregated image overlayed, together
         with the aligned area and the aligned_box shown as blue rectangles.
         aligned_he will contain the part of the H&E image aligned.
     :rtype: (numpy.ndarray, numpy.ndarray)
@@ -445,35 +478,29 @@ def align_he_aggregated_img(
 
     h, w = img.shape
 
-    bottom_right = (int(w * ((1+box_size)/2)),
-                    int(h * ((1+box_size)/2)))
-    top_left = (int(w * ((1-box_size) / 2)),
-                int(h * ((1-box_size) / 2)))
+    bottom_right = (int(w * ((1 + box_size) / 2)), int(h * ((1 + box_size) / 2)))
+    top_left = (int(w * ((1 - box_size) / 2)), int(h * ((1 - box_size) / 2)))
 
-    img= img_bw[
-        top_left[1]:bottom_right[1],
-        top_left[0]:bottom_right[0]    
-    ]
+    img = img_bw[top_left[1] : bottom_right[1], top_left[0] : bottom_right[0]]
 
     highest_cor, he_res, he_orig, tl, br = align_he_img(
-        he_path,
-        expression_img=img,
-        bw_threshold=bw_threshold,
-        use_bw = True
+        he_path, expression_img=img, bw_threshold=bw_threshold, use_bw=True
     )
 
-    align_w = br[0]-tl[0]
-    align_h = br[1]-tl[1]
-    multiplier = (1-box_size) / (box_size * 2)
-    box_tl = (int(tl[0]-align_w * multiplier), int(tl[1]-align_h * multiplier))
-    #box_tl = h,w
-    box_br = (int(br[0]+align_w * multiplier), int(br[1]+align_h * multiplier))
+    align_w = br[0] - tl[0]
+    align_h = br[1] - tl[1]
+    multiplier = (1 - box_size) / (box_size * 2)
+    box_tl = (int(tl[0] - align_w * multiplier), int(tl[1] - align_h * multiplier))
+    # box_tl = h,w
+    box_br = (int(br[0] + align_w * multiplier), int(br[1] + align_h * multiplier))
 
     he_res_ratio = he_res.shape[1] / he_res.shape[0]
 
-    he_orig = cv2.resize(he_orig,
-                     (int(he_orig.shape[0] * he_res_ratio),he_orig.shape[0]),
-                     cv2.INTER_AREA)
+    he_orig = cv2.resize(
+        he_orig,
+        (int(he_orig.shape[0] * he_res_ratio), he_orig.shape[0]),
+        cv2.INTER_AREA,
+    )
 
     horizontal_ratio = he_orig.shape[0] / he_res.shape[0]
     vertical_ratio = he_orig.shape[1] / he_res.shape[1]
@@ -481,34 +508,33 @@ def align_he_aggregated_img(
     # scale boxes
     tl = int(tl[0] * horizontal_ratio), int(tl[1] * vertical_ratio)
     br = int(br[0] * horizontal_ratio), int(br[1] * vertical_ratio)
-    box_tl = (int(box_tl[0] * horizontal_ratio),
-              int(box_tl[1] * vertical_ratio))
-    box_br = (int(box_br[0] * horizontal_ratio),
-              int(box_br[1] * vertical_ratio))
+    box_tl = (int(box_tl[0] * horizontal_ratio), int(box_tl[1] * vertical_ratio))
+    box_br = (int(box_br[0] * horizontal_ratio), int(box_br[1] * vertical_ratio))
 
-    img_resized = cv2.resize(img_bw,
-                                  (box_br[0] - box_tl[0], box_br[1] - box_tl[1]),
-                                   cv2.INTER_NEAREST)
+    img_resized = cv2.resize(
+        img_bw, (box_br[0] - box_tl[0], box_br[1] - box_tl[1]), cv2.INTER_NEAREST
+    )
 
     align_bw = cv2.cvtColor(img_resized, cv2.COLOR_GRAY2BGR)
 
-    he_align = he_orig[box_tl[1]:box_br[1], box_tl[0]:box_br[0]].copy()
+    he_align = he_orig[box_tl[1] : box_br[1], box_tl[0] : box_br[0]].copy()
+
+    cv2.rectangle(he_orig, tl, br, (180, 233, 86, 255), 5)
+    cv2.rectangle(he_orig, box_tl, box_br, (180, 233, 86, 255), 5)
 
-    cv2.rectangle(he_orig, tl, br,  (180, 233, 86, 255), 5)
-    cv2.rectangle(he_orig, box_tl, box_br,  (180, 233, 86, 255), 5)
-    
     overlay = he_orig.copy()
-    overlay[box_tl[1]:box_br[1], box_tl[0]:box_br[0]] &= align_bw
-    
+    overlay[box_tl[1] : box_br[1], box_tl[0] : box_br[0]] &= align_bw
+
     cv2.addWeighted(overlay, 0.4, he_orig, 0.6, 0, he_orig)
-    
+
     return he_orig, he_align
 
+
 def attach_he_adata(
-    adata : anndata.AnnData,
-    aligned_he : numpy.ndarray,
-    push_by_spot_diameter : bool=True,
-    raw_aligned : bool=False
+    adata: anndata.AnnData,
+    aligned_he: numpy.ndarray,
+    push_by_spot_diameter: bool = True,
+    raw_aligned: bool = False,
 ) -> anndata.AnnData:
     """
     Attach the aligned H&E data to a spacemake processed AnnData object.
@@ -518,74 +544,83 @@ def attach_he_adata(
     :param aligned_he: A spacemake aligned H&E image aligned either with the
         align_he_aggregated_img or the align_he_spot_img functions.
     :type aligned_he: numpy.ndarray
-    :param push_by_spot_diameter: If true the aligned image will be pushed by 
-        one spot diameter. To be used only if a spot image was aligned. 
+    :param push_by_spot_diameter: If true the aligned image will be pushed by
+        one spot diameter. To be used only if a spot image was aligned.
         Defaults to True.
     :type push_by_spot_diameter: bool
-    :param raw_aligned: If true it is assumed that the H&E image was aligned 
-        using the raw spacemake processed AnnData. If a spot image was aligned 
+    :param raw_aligned: If true it is assumed that the H&E image was aligned
+        using the raw spacemake processed AnnData. If a spot image was aligned
         this should be set to False. Defaults to False.
     :type raw_aligned: bool
-    :returns: An AnnData object with the attached H&E image. When used with 
+    :returns: An AnnData object with the attached H&E image. When used with
         scanpy's scanpy.pl.spatial, under the spatial data the H&E image will
         be shown.
     :rtype: anndata.AnnData
     """
     import math
+
     # get the width of the puck
-    puck_width_um = adata.uns['puck_variables']['width_um']
-    spot_diameter_um = adata.uns['puck_variables']['spot_diameter_um']
-    
-    if adata.uns['run_mode_variables']['mesh_data']:
-        spot_diameter_um = math.sqrt(3) * adata.uns['run_mode_variables']['mesh_spot_diameter_um']
-    
+    puck_width_um = adata.uns["puck_variables"]["width_um"]
+    spot_diameter_um = adata.uns["puck_variables"]["spot_diameter_um"]
+
+    if adata.uns["run_mode_variables"]["mesh_data"]:
+        spot_diameter_um = (
+            math.sqrt(3) * adata.uns["run_mode_variables"]["mesh_spot_diameter_um"]
+        )
+
     # get the width in coordinates
-    width_coord_max = adata.uns['puck_variables']['width_um'] * \
-        adata.uns['puck_variables']['coord_by_um']
-    height_coord_max = adata.uns['puck_variables']['height_um'] * \
-        adata.uns['puck_variables']['coord_by_um']
+    width_coord_max = (
+        adata.uns["puck_variables"]["width_um"]
+        * adata.uns["puck_variables"]["coord_by_um"]
+    )
+    height_coord_max = (
+        adata.uns["puck_variables"]["height_um"]
+        * adata.uns["puck_variables"]["coord_by_um"]
+    )
 
-    spot_distance_um = 1 / adata.uns['puck_variables']['coord_by_um']
+    spot_distance_um = 1 / adata.uns["puck_variables"]["coord_by_um"]
 
     # get the width of the processed adata
-    adata_width_um = spot_distance_um *\
-        (adata.obsm['spatial'].max(axis=0)[0])
+    adata_width_um = spot_distance_um * (adata.obsm["spatial"].max(axis=0)[0])
 
     # rotate he to align with coordinate system of scanpy
     rotated_he = cv2.flip(cv2.rotate(aligned_he, cv2.ROTATE_90_CLOCKWISE), 1)
-    
+
     h_px, w_px = rotated_he.shape[:2]
 
-    px_per_um = w_px / adata_width_um 
+    px_per_um = w_px / adata_width_um
 
     # convert so that origin at 0, 0
-    locations = adata.obsm['spatial']
-    
+    locations = adata.obsm["spatial"]
+
     if not raw_aligned:
         locations = locations - locations.min(axis=0)
 
     spot_diameter_px = spot_diameter_um * px_per_um
 
     if raw_aligned:
-        
-        locations = locations * [(w_px ) / width_coord_max,
-                             (h_px ) / height_coord_max]
+
+        locations = locations * [(w_px) / width_coord_max, (h_px) / height_coord_max]
     else:
-        locations = locations * [(w_px ) / locations.max(axis=0)[0],
-                             (h_px ) / locations.max(axis=0)[1]]
-    
+        locations = locations * [
+            (w_px) / locations.max(axis=0)[0],
+            (h_px) / locations.max(axis=0)[1],
+        ]
+
     if push_by_spot_diameter:
         locations = [spot_diameter_px, spot_diameter_px] + locations
-        
-    locations = locations.astype('int')
-
-    adata.uns['spatial']={
-        'img': {'images': {
-                    'hires': rotated_he
-                    },
-                'scalefactors': {'tissue_hires_scalef': 1,
-                                 'spot_diameter_fullres': spot_diameter_px}} 
+
+    locations = locations.astype("int")
+
+    adata.uns["spatial"] = {
+        "img": {
+            "images": {"hires": rotated_he},
+            "scalefactors": {
+                "tissue_hires_scalef": 1,
+                "spot_diameter_fullres": spot_diameter_px,
+            },
+        }
     }
-    adata.obsm['spatial'] = locations
-    
+    adata.obsm["spatial"] = locations
+
     return adata
diff --git a/spacemake/spatial/novosparc_integration.py b/spacemake/spatial/novosparc_integration.py
index da721d73..afea3500 100644
--- a/spacemake/spatial/novosparc_integration.py
+++ b/spacemake/spatial/novosparc_integration.py
@@ -5,86 +5,14 @@
 from spacemake.util import message_aggregation
 from numpy import ndarray
 
-logger_name = 'spacemake.spatial.novosparc_integration'
+logger_name = "spacemake.spatial.novosparc_integration"
 logger = logging.getLogger(logger_name)
 
-def get_parser():
-    """get_parser."""
-    import argparse
-
-    parser = argparse.ArgumentParser(
-        allow_abbrev=False,
-        description='reconstruct 2-d tissue with novosparc',
-    )
-
-    parser.add_argument(
-        '--single_cell_dataset', type=str, help='path to single_cell.h5ad file',
-        required = True,
-    )
-
-    parser.add_argument(
-        '--spatial_dataset', type=str, help='path to spatial.h5ad file',
-        required=False,
-    )
-
-    parser.add_argument(
-        '--output', type=str, help='path to output.h5ad file',
-        required=True,
-    )
-
-    return parser
-
-def get_spacemake_parser(parent_parser_subparsers):
-    parser_novosparc = parent_parser_subparsers.add_parser(
-        'novosparc',
-        help='reconstruct the 2-d tissue with novosparc',
-        parents=[
-            get_run_parser(),
-            get_project_sample_parser(
-                    allow_multiple = False,
-                    help_extra = ' of sample to reconstruct',
-                 ),
-        ],
-    )
-        
-    parser_novosparc.add_argument(
-        '--run_mode', type=str, help='the run_mode to be used for this sample ' +
-        'for reconstruction. If left empty, the first run_mode for this ' +
-        'sample will be used', required=False,
-    )
-
-    parser_novosparc.add_argument(
-        '--umi_cutoff', type=int, help='umi_cutoff to be used for this sample ' +
-        'for reconstruction. If left empty, the smallest umi_cutoff of a given ' +
-        'run_mode will be used', required=False,
-    )
-
-    parser_novosparc.add_argument(
-        '--reference_project_id', type=str, help='project_id of the reference atlas.' +
-        'Has to be spatial, otherwise error will be raised', required=False,
-    )
-
-    parser_novosparc.add_argument(
-        '--reference_sample_id', type=str, help='sample_id of the reference atlas.' +
-        'Has to be spatial, otherwise error will be raised', required=False,
-    )
-
-    parser_novosparc.add_argument(
-        '--reference_run_mode', type=str, help='the run_mode to be used for the ' +
-        'reference sample. If empty, the first run_mode for the reference will be used',
-        required=False,
-    )
-
-    parser_novosparc.add_argument(
-        '--reference_umi_cutoff', type=str, help='the umi_cutoff to be used for this ' +
-        'reference sample. If empty, the smallest umi_cutoff of the given run_mode ' +
-        'will be used.', required=False,
-    )
-
-    return parser_novosparc
 
 # copy of a yet-to-be-pushed novosparc function
-def quantify_clusters_spatially(tissue: novosparc.cm.Tissue, cluster_key: str='clusters') -> ndarray:
+def quantify_clusters_spatially(
+    tissue: novosparc.cm.Tissue, cluster_key: str = "clusters"
+) -> ndarray:
     """Maps the annotated clusters obtained from the scRNA-seq analysis onto
     the tissue space.
 
@@ -102,24 +30,36 @@ def quantify_clusters_spatially(tissue: novosparc.cm.Tissue, cluster_key: str='c
     clusters = tissue.dataset.obs[cluster_key].to_numpy().flatten()
     cluster_names = np.unique(clusters)
     ixs = np.array(
-            [np.argmax(
+        [
+            np.argmax(
                 np.array(
-                    [np.median(
-                        np.array(
-                            tissue.gw[:, location][np.argwhere(clusters == cluster).flatten()]))
-                                         for cluster in cluster_names]))
-                    for location in range(len(tissue.locations))])
+                    [
+                        np.median(
+                            np.array(
+                                tissue.gw[:, location][
+                                    np.argwhere(clusters == cluster).flatten()
+                                ]
+                            )
+                        )
+                        for cluster in cluster_names
+                    ]
+                )
+            )
+            for location in range(len(tissue.locations))
+        ]
+    )
     return [cluster_names[ix] for ix in ixs]
-    
+
+
 def novosparc_denovo(
-        adata: anndata.AnnData,
-        num_spatial_locations: int=5000,
-        num_input_cells: int=30000,
-        locations=None,
-    ) -> novosparc.cm.Tissue:
+    adata: anndata.AnnData,
+    num_spatial_locations: int = 5000,
+    num_input_cells: int = 30000,
+    locations=None,
+) -> novosparc.cm.Tissue:
     """
     Given an AnnData object containing single-cell data, this function
-    will try to reconstruct a 2D tissue de-novo. By default it will use 
+    will try to reconstruct a 2D tissue de-novo. By default it will use
     the 500 most variable genes as markers to do this reconstruction.
 
     :param adata: A spacemake processed sample.
@@ -135,8 +75,8 @@ def novosparc_denovo(
         are used. Default: 30000
     :type num_input_cells: int
     :param locations: Numpy array of (x,y) coordinates (optional).
-        Novosparc will try to reconstruct the tissue using these 
-        locations. If none provided, a circular tissue will be 
+        Novosparc will try to reconstruct the tissue using these
+        locations. If none provided, a circular tissue will be
         automatically created with num_spatial_locations number of
         locations.
     :type locations: numpy.ndarray
@@ -152,8 +92,8 @@ def novosparc_denovo(
 
     from scipy.sparse import issparse, csc_matrix
 
-    logger.info('Reconstructing the tissue de-novo with novosparc') 
-    
+    logger.info("Reconstructing the tissue de-novo with novosparc")
+
     gene_names = adata.var.index.tolist()
 
     num_cells, num_genes = adata.shape
@@ -165,13 +105,13 @@ def novosparc_denovo(
         num_spatial_locations = num_cells
 
     if num_cells > num_input_cells:
-        sc.pp.subsample(adata, n_obs = num_input_cells)
+        sc.pp.subsample(adata, n_obs=num_input_cells)
 
     if num_cells < num_spatial_locations:
         num_spatial_locations = num_cells
 
-    sc.pp.highly_variable_genes(adata, n_top_genes = 100)
-    is_var_gene = adata.var['highly_variable']
+    sc.pp.highly_variable_genes(adata, n_top_genes=100)
+    is_var_gene = adata.var["highly_variable"]
     # select only 100 genes
     var_genes = list(is_var_gene.index[is_var_gene])
 
@@ -180,29 +120,33 @@ def novosparc_denovo(
     # if you uncomment this, adata.X will be stored not as sparse matrix, rather
     # as a regular numpy array. uncommenting this doesnt throw an error
     if issparse(adata.X):
-        dense_adata = anndata.AnnData(
-            adata.X.toarray(),
-            obs = adata.obs,
-            var = adata.var)
+        dense_adata = anndata.AnnData(adata.X.toarray(), obs=adata.obs, var=adata.var)
         adata = dense_adata
         del dense_adata
 
     if locations is None:
         # create circle locations
-        locations = novosparc.gm.construct_circle(num_locations = num_spatial_locations)
+        locations = novosparc.gm.construct_circle(num_locations=num_spatial_locations)
 
     tissue = novosparc.cm.Tissue(dataset=adata, locations=locations)
 
     num_neighbors_s = num_neighbors_t = 5
 
-    logger.info('Novosparc setup')
-    tissue.setup_smooth_costs(dge_rep=dge_rep, num_neighbors_s=num_neighbors_s, num_neighbors_t=num_neighbors_t)
+    logger.info("Novosparc setup")
+    tissue.setup_smooth_costs(
+        dge_rep=dge_rep,
+        num_neighbors_s=num_neighbors_s,
+        num_neighbors_t=num_neighbors_t,
+    )
 
     tissue.reconstruct(alpha_linear=0, epsilon=5e-3)
 
     return tissue
 
-def novosparc_mapping(sc_adata: anndata.AnnData, st_adata: anndata.AnnData) -> novosparc.cm.Tissue:
+
+def novosparc_mapping(
+    sc_adata: anndata.AnnData, st_adata: anndata.AnnData
+) -> novosparc.cm.Tissue:
     """
     Given two AnnData objects, one single-cell and one spatial, this function
     will map the expression of the single-cell data onto the spatial data using
@@ -213,7 +157,7 @@ def novosparc_mapping(sc_adata: anndata.AnnData, st_adata: anndata.AnnData) -> n
     :param st_adata: A spacemake processed spatial sample.
     :type st_adata: anndata.AnnData
     :returns: A novosparc.cm.Tissue object with 2D expression information.
-        The locations of the Tissue will be identical to the locations of 
+        The locations of the Tissue will be identical to the locations of
         the spatial sample.
     :rtype: novosparc.cm.Tissue
     """
@@ -224,14 +168,14 @@ def novosparc_mapping(sc_adata: anndata.AnnData, st_adata: anndata.AnnData) -> n
     from scanpy._utils import check_nonnegative_integers
     from scipy.sparse import csc_matrix
 
-    logger.info('Mapping single-cell data onto spatial data with novosparc')
+    logger.info("Mapping single-cell data onto spatial data with novosparc")
 
-    if (check_nonnegative_integers(sc_adata.X)
-        or check_nonnegative_integers(st_adata.X)
-    ):
+    if check_nonnegative_integers(sc_adata.X) or check_nonnegative_integers(st_adata.X):
         # if any of the inputs is count-data, raise error
-        raise SpacemakeError(f'External dge seems to contain raw counts. '+
-            'Normalised values are expected for both sc_adata and st_adata.')
+        raise SpacemakeError(
+            f"External dge seems to contain raw counts. "
+            + "Normalised values are expected for both sc_adata and st_adata."
+        )
 
     # calculate top 500 variable genes for both
     sc.pp.highly_variable_genes(sc_adata, n_top_genes=500)
@@ -242,50 +186,57 @@ def novosparc_mapping(sc_adata: anndata.AnnData, st_adata: anndata.AnnData) -> n
 
     markers = list(set(sc_adata_hv).intersection(st_adata_hv))
 
-    logger.info(f'{len(markers)} number of common markers found. Using them' +
-        ' for reconstruction')
+    logger.info(
+        f"{len(markers)} number of common markers found. Using them"
+        + " for reconstruction"
+    )
 
     # save sc dge as a pandas dataframe
     dge_rep = sc_adata.to_df()[sc_adata_hv]
 
-    if not 'spatial' in st_adata.obsm:
-        raise SpacemakeError(f'The object provided to st_adata is not spatial')
+    if not "spatial" in st_adata.obsm:
+        raise SpacemakeError(f"The object provided to st_adata is not spatial")
 
-    locations = st_adata.obsm['spatial']
+    locations = st_adata.obsm["spatial"]
     atlas_matrix = st_adata.to_df()[markers].values
 
     # make dense dataset
     dense_dataset = anndata.AnnData(
-        sc_adata.X.toarray(),
-        obs = sc_adata.obs,
-        var = sc_adata.var)
+        sc_adata.X.toarray(), obs=sc_adata.obs, var=sc_adata.var
+    )
 
     marker_ix = [dense_dataset.var.index.get_loc(marker) for marker in markers]
 
     tissue = novosparc.cm.Tissue(dataset=dense_dataset, locations=locations)
     num_neighbors_s = num_neighbors_t = 5
 
-    tissue.setup_linear_cost(markers_to_use=marker_ix, atlas_matrix=atlas_matrix,
-                             markers_metric='minkowski', markers_metric_p=2)
-    tissue.setup_smooth_costs(dge_rep = dge_rep,
-                              num_neighbors_s=num_neighbors_s,
-                              num_neighbors_t=num_neighbors_t)
+    tissue.setup_linear_cost(
+        markers_to_use=marker_ix,
+        atlas_matrix=atlas_matrix,
+        markers_metric="minkowski",
+        markers_metric_p=2,
+    )
+    tissue.setup_smooth_costs(
+        dge_rep=dge_rep,
+        num_neighbors_s=num_neighbors_s,
+        num_neighbors_t=num_neighbors_t,
+    )
 
     tissue.reconstruct(alpha_linear=0.5, epsilon=5e-3)
 
     return tissue
 
+
 def save_novosparc_res(
-        tissue : novosparc.cm.Tissue,
-        adata_original : anndata.AnnData
-    ) -> anndata.AnnData:
+    tissue: novosparc.cm.Tissue, adata_original: anndata.AnnData
+) -> anndata.AnnData:
     """Save the result of novosparc spatial mapping.
 
     :param tissue: A Tissue object, result of the novosparc mapping.
     :type tissue: novosparc.cm.Tissue
     :param adata_original: The original AnnData which was mapped with novosparc,
-        to reconstruct the Tissue. For the de-novo reconstruction, use the 
-        original AnnData object, when a single-cell dataset was mapped onto a 
+        to reconstruct the Tissue. For the de-novo reconstruction, use the
+        original AnnData object, when a single-cell dataset was mapped onto a
         spatial dataset, use the AnnData of the single-cell dataset.
     :type adata_original: anndata.AnnData
     :returns: An AnnData object containing the spatial information in
@@ -300,28 +251,37 @@ def save_novosparc_res(
     from scipy.sparse import csc_matrix
 
     adata_reconst = anndata.AnnData(
-        csc_matrix(tissue.sdge.T),
-        var = pd.DataFrame(index=tissue.dataset.var_names))
+        csc_matrix(tissue.sdge.T), var=pd.DataFrame(index=tissue.dataset.var_names)
+    )
 
-    logger.info('Scaling mapped data to original data...')
+    logger.info("Scaling mapped data to original data...")
 
-    adata_reconst.X = np.sum(adata_original.X) * adata_reconst.X / np.sum(adata_reconst.X)
+    adata_reconst.X = (
+        np.sum(adata_original.X) * adata_reconst.X / np.sum(adata_reconst.X)
+    )
 
-    adata_reconst.obsm['spatial'] = tissue.locations
+    adata_reconst.obsm["spatial"] = tissue.locations
 
-    logger.info('Transferring original cluster labels...')
+    logger.info("Transferring original cluster labels...")
 
-    for res_key in adata_original.obs.columns[adata_original.obs.columns.str.startswith('leiden_')]:
-        adata_reconst.obs[f'novosparc_{res_key}'] = quantify_clusters_spatially(tissue, res_key)
+    for res_key in adata_original.obs.columns[
+        adata_original.obs.columns.str.startswith("leiden_")
+    ]:
+        adata_reconst.obs[f"novosparc_{res_key}"] = quantify_clusters_spatially(
+            tissue, res_key
+        )
 
     return adata_reconst
 
+
 @message_aggregation(logger_name)
 def cmdline():
     """cmdline."""
     import scanpy as sc
 
-    parser = get_parser()
+    from spacemake.cmdline import get_novosparc_parser
+
+    parser = get_novosparc_parser()
 
     args = parser.parse_args()
 
@@ -334,10 +294,11 @@ def cmdline():
         tissue_reconst = novosparc_mapping(sc_adata, st_adata)
         adata = save_novosparc_res(tissue_reconst, sc_adata)
     else:
-        tissue_reconst = novosparc_denovo(sc_adata)  
+        tissue_reconst = novosparc_denovo(sc_adata)
         adata = save_novosparc_res(tissue_reconst, sc_adata)
 
     adata.write(args.output)
 
+
 if __name__ == "__main__":
     cmdline()
diff --git a/spacemake/spatial/util.py b/spacemake/spatial/util.py
index 1b25f8f3..98d22bc3 100644
--- a/spacemake/spatial/util.py
+++ b/spacemake/spatial/util.py
@@ -1,4 +1,5 @@
-from spacemake.preprocess import calculate_adata_metrics
+# from anndata import AnnData
+from spacemake.preprocess.dge import calculate_adata_metrics
 
 
 def compute_neighbors(adata, min_dist=None, max_dist=None):
diff --git a/spacemake/util.py b/spacemake/util.py
index 6aa3fdc2..7fdedf40 100644
--- a/spacemake/util.py
+++ b/spacemake/util.py
@@ -2,13 +2,70 @@
 import os
 import logging
 
-from contextlib import ContextDecorator, contextmanager
+#from contextlib import ContextDecorator, contextmanager
 from spacemake.errors import SpacemakeError, FileWrongExtensionError
+from spacemake.contrib import __version__, __license__, __author__, __email__
 
 LINE_SEPARATOR = "-" * 50 + "\n"
 
 bool_in_str = ["True", "true", "False", "false"]
 
+def generate_kmers(k, nts='ACGT'):
+    if k == 0:
+        yield ''
+    elif k > 0:
+        for x in nts:
+            for mer in generate_kmers(k-1, nts=nts):
+                yield x + mer
+
+class dotdict(dict):
+    """dot.notation access to dictionary attributes"""
+
+    __getattr__ = dict.get
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+    def __str__(self):
+        buf = ["dotdict"]
+        for k, v in self.items():
+            if not k.startswith("__"):
+                buf.append(f"  {k} = {v}")
+
+        return "\n".join(buf)
+
+
+def wc_fill(x, wc):
+    """
+    Versatile snakemake helper function that can render any string template used in the mapping and reports modules,
+    either filling in from a Wildcards object, or from a dotdict.
+    """
+    return x.format(
+        sample_id=getattr(wc, "sample_id", "NO_sample_id"),
+        project_id=getattr(wc, "project_id", "NO_project_id"),
+        species=getattr(wc, "species", "NO_species"),
+        ref_name=getattr(wc, "ref_name", "NO_ref_name"),
+        mapper=getattr(wc, "mapper", "NO_mapper"),
+        link_name=getattr(wc, "link_name", "NO_link_name"),
+        polyA_adapter_trimmed=getattr(wc, "polyA_adapter_trimmed", ""),
+        data_root_type=getattr(wc, "data_root_type", "complete_data"),
+    )
+
+def quiet_bam_open(*argc, **kw):
+    """_summary_
+
+    This wrapper around pysam.AlignmentFile() simply silences warnings about missing BAM index etc.
+    We don't care about the index and therefore these error messages are just spam.
+
+    Returns:
+        pysam.AlignmentFile: the sam/bam object as returned by pysam.
+    """
+    import pysam
+
+    save = pysam.set_verbosity(0)
+    bam = pysam.AlignmentFile(*argc, **kw)
+    pysam.set_verbosity(save)
+    return bam
+
 
 def assert_file(file_path, default_value=None, extension=["all"]):
     if file_path == default_value:
@@ -50,12 +107,43 @@ def str2bool(var):
 
 
 def ensure_path(path):
-    import os
+    dirname = os.path.dirname(path)
+    if dirname:
+        os.makedirs(dirname, exist_ok=True)
+    return path
 
+def timed_loop(
+    src,
+    logger,
+    T=5,
+    chunk_size=10000,
+    template="processed {i} BAM records in {dT:.1f}sec. ({rate:.3f} k rec/sec)",
+    skim=0,
+):
+    from time import time
+
+    t0 = time()
+    t_last = t0
+    i = 0
+    for i, x in enumerate(src):
+        if skim:
+            if i % skim == 0:
+                yield x
+        else:
+            yield x
 
-def ensure_path(path):
-    os.makedirs(os.path.dirname(path), exist_ok=True)
-    return path
+        if i % chunk_size == 0:
+            t = time()
+            if t - t_last > T:
+                dT = t - t0
+                rate = i / dT / 1000.0
+                logger.info(template.format(**locals()))
+                t_last = t
+
+    t = time()
+    dT = t - t0
+    rate = i / dT / 1000.0
+    logger.info("Finished! " + template.format(**locals()))
 
 
 def FASTQ_src(src):
@@ -68,7 +156,7 @@ def FASTQ_src(src):
 def BAM_src(src):
     import pysam
 
-    bam = pysam.AlignmentFile(src, "rb", check_sq=False)
+    bam = quiet_bam_open(src, "rb", check_sq=False)
     for read in bam.fetch(until_eof=True):
         yield read.query_name, read.query_sequence, read.query_qualities
 
@@ -77,6 +165,20 @@ def read_fq(fname, skim=0):
     import gzip
 
     logger = logging.getLogger("spacemake.util.read_fq")
+    if str(fname) == "None":
+        logger.warning("yielding empty data forever")
+        while True:
+            yield ("no_qname", "no_seq", "no_qual")
+
+    if "*" in fname:
+        logger.warning("EXPERIMENTAL: fname contains wildcards")
+        from glob import glob
+
+        for match in sorted(glob(fname)):
+            for rec in read_fq(match, skim=skim):
+                yield rec
+
+    logger.info(f"iterating over reads from '{fname}'")
     if fname.endswith(".gz"):
         src = FASTQ_src(gzip.open(fname, mode="rt"))
     elif fname.endswith(".bam"):
@@ -86,14 +188,36 @@ def read_fq(fname, skim=0):
     else:
         src = FASTQ_src(fname)  # assume its a stream or file-like object already
 
-    for i, (name, seq, qual) in enumerate(src):
-        if not skim:
-            yield name, seq, qual
-        else:
-            if (i % skim) == 0:
-                yield name, seq, qual
+    n = 0
+    for record in timed_loop(src, logger, T=15, template="processed {i} reads in {dT:.1f}sec. ({rate:.3f} k rec/sec)", skim=skim):
+        yield record
+        n += 1
 
-    logger.info(f"processed {i} FASTQ records from '{fname}'")
+    logger.info(f"processed {n} FASTQ records from '{fname}'")
+
+
+def make_header(bam, progname):
+    import os
+    import sys
+
+    header = bam.header.to_dict()
+    # if "PG" in header:
+    # for pg in header['PG']:
+    #     if pg["ID"] == progname:
+    #         progname = progname + ".1"
+
+    pg_list = header.get("PG", [])
+    pg = {
+        "ID": progname,
+        "PN": progname,
+        "CL": " ".join(sys.argv[1:]),
+        "VN": __version__,
+    }
+    if len(pg_list):
+        pg["PP"] = pg_list[-1]["ID"]
+
+    header["PG"] = pg_list + [pg]
+    return header
 
 
 def dge_to_sparse(dge_path):
@@ -246,32 +370,67 @@ def fasta_chunks(lines, strip=True, fuse=True):
         yield chunk, "".join(data)
 
 
-@contextmanager
-def message_aggregation(log_listen="spacemake", print_logger=False, print_success=True):
-    message_buffer = []
+# @contextmanager
+# def message_aggregation(log_listen="spacemake", print_logger=False, print_success=True):
+#     message_buffer = []
 
-    log = logging.getLogger(log_listen)
-    log.setLevel(logging.INFO)
+#     log = logging.getLogger(log_listen)
+#     log.setLevel(logging.INFO)
 
-    class MessageHandler(logging.NullHandler):
-        def handle(this, record):
-            if record.name == log_listen:
-                if print_logger:
-                    print(f"{log_listen}: {record.msg}")
-                else:
-                    print(record.msg)
+#     class MessageHandler(logging.NullHandler):
+#         def handle(this, record):
+#             if record.name == log_listen:
+#                 if print_logger:
+#                     print(f"{log_listen}: {record.msg}")
+#                 else:
+#                     print(record.msg)
+
+#     log.addHandler(MessageHandler())
 
-    log.addHandler(MessageHandler())
+#     try:
+#         yield True
 
-    try:
-        yield True
+#         if print_success:
+#             print(f"{LINE_SEPARATOR}SUCCESS!")
 
-        if print_success:
-            print(f"{LINE_SEPARATOR}SUCCESS!")
+#     except SpacemakeError as e:
+#         print(e)
+
+def message_aggregation(log_listen="spacemake", print_logger=False, print_success=True):
+    from functools import wraps
+
+    def the_decorator(func):
+        @wraps(func)
+        def wrapper(*argc, **kw):
+            message_buffer = []
+
+            log = logging.getLogger(log_listen)
+            log.setLevel(logging.INFO)
+
+            class MessageHandler(logging.NullHandler):
+                def handle(this, record):
+                    if record.name == log_listen:
+                        if print_logger:
+                            print(f"{log_listen}: {record.msg}")
+                        else:
+                            print(record.msg)
+
+            log.addHandler(MessageHandler())
+
+            try:
+                res = func(*argc, **kw)
+            except (SpacemakeError, NotImplementedError) as err:
+                print(f"CAUGHT A SPACEMAKE ERROR {err}")
+                # print(err)
+                return err
+            else:
+                if print_success:
+                    print(f"{LINE_SEPARATOR}SUCCESS!")
+                return res
 
-    except SpacemakeError as e:
-        print(e)
+        return wrapper        
 
+    return the_decorator
 
 def str_to_list(value):
     # if list in string representation, return the list
@@ -302,3 +461,120 @@ def check_star_index_compatibility(star_index_dir):
                 f"STAR index version ({index_version}) is"
                 + f" incompatible with your STAR version ({star_version})"
             )
+
+
+def load_yaml(path, mode="rt"):
+    if not path:
+        return {}
+    else:
+        import yaml
+
+        return yaml.load(path, mode)
+
+
+def setup_logging(
+    args,
+    name="spacemake.main",
+    log_file="",
+    FORMAT="%(asctime)-20s\t{sample:30s}\t%(name)-50s\t%(levelname)s\t%(message)s",
+):
+    sample = getattr(args, "sample", "na")
+    import setproctitle
+    if name != "spacemake.main":
+        setproctitle.setproctitle(f"{name} {sample}")
+
+    FORMAT = FORMAT.format(sample=sample)
+
+    log_level = getattr(args, "log_level", "INFO")
+    lvl = getattr(logging, log_level)
+    logging.basicConfig(level=lvl, format=FORMAT)
+    root = logging.getLogger("spacemake")
+    root.setLevel(lvl)
+
+    log_file = getattr(args, "log_file", log_file)
+    if log_file:
+        fh = logging.FileHandler(filename=ensure_path(log_file), mode="a")
+        fh.setFormatter(logging.Formatter(FORMAT))
+        root.debug(f"adding log-file handler '{log_file}'")
+        root.addHandler(fh)
+
+    if hasattr(args, "debug"):
+        # cmdline requested debug output for specific domains (comma-separated)
+        for logger_name in args.debug.split(","):
+            if logger_name:
+                root.info(f"setting domain {logger_name} to DEBUG")
+                logging.getLogger(logger_name.replace("root", "")).setLevel(
+                    logging.DEBUG
+                )
+
+    logger = logging.getLogger(name)
+    logger.debug("started logging")
+    for k, v in sorted(vars(args).items()):
+        logger.debug(f"cmdline arg\t{k}={v}")
+
+    return logger
+
+def setup_smk_logging(name="spacemake.smk", **kw):
+    import argparse
+    args = argparse.Namespace(**kw)
+    return setup_logging(args, name=name)
+
+default_log_level = "INFO"
+
+
+def make_minimal_parser(prog="", usage="", **kw):
+    import argparse
+
+    parser = argparse.ArgumentParser(prog=prog, usage=usage, **kw)
+    parser.add_argument(
+        "--log-file",
+        default=f"{prog}.log",
+        help=f"place log entries in this file (default={prog}.log)",
+    )
+    parser.add_argument(
+        "--log-level",
+        default=default_log_level,
+        help=f"change threshold of python logging facility (default={default_log_level})",
+    )
+    parser.add_argument(
+        "--debug",
+        default="",
+        help=f"comma-separated list of logging-domains for which you want DEBUG output",
+    )
+    parser.add_argument(
+        "--sample", default="sample_NA", help="sample_id (where applicable)"
+    )
+    return parser
+
+
+def load_config_with_fallbacks(args, try_yaml="config.yaml"):
+    """
+    Tries to load spacemake configuration from
+        1) args.config
+        2) try_yaml
+        3) builtin default from spacamake package
+
+    The entire configuration is attached to the args namespace such that
+    args.config["barcode_flavors"]["dropseq] can work.
+    """
+    from spacemake.config import ConfigFile
+
+    config = None
+    if hasattr(args, "config") and os.access(args.config, os.R_OK):
+        config = ConfigFile.from_yaml(args.config)
+    elif os.access(try_yaml, os.R_OK):
+        config = ConfigFile.from_yaml(try_yaml)
+    else:
+        builtin = os.path.join(os.path.dirname(__file__), "data/config/config.yaml")
+        config = ConfigFile.from_yaml(builtin)
+
+    args_kw = vars(args)
+    # try:
+    #     args_kw["log_level"] = config.variables["logging"]["level"]
+    # except KeyError:
+    #     pass
+
+    args_kw["config"] = config.variables
+    import argparse
+
+    return argparse.Namespace(**args_kw)


Test patch:
diff --git a/test_data/README.md b/test_data/README.md
new file mode 100644
index 00000000..073963b6
--- /dev/null
+++ b/test_data/README.md
@@ -0,0 +1,8 @@
+# SPACEMAKE test data 
+
+There is a mix of old and new stuff in here that needs to be cleaned up. In general, I think file size should not exceed 1 MB. If you need a larger file, let's place it on bimsbstatic and download it from there like so:
+
+`wget https://bimsbstatic.mdc-berlin.de/rajewsky/spacemake-test-data/spacemake_tile_test_data.tar.gz`
+
+
+Thanks!
diff --git a/test_data/make_chr22_test_data.py b/test_data/make_chr22_test_data.py
new file mode 100644
index 00000000..3af55884
--- /dev/null
+++ b/test_data/make_chr22_test_data.py
@@ -0,0 +1,156 @@
+import pysam
+import pandas as pd
+from spacemake.util import FASTQ_src, read_fq
+from byo.track import load_track
+
+genome = load_track("/data/rajewsky/genomes/hg38/hg38.fa")
+
+# load output of gene_loci_to_gtf.py: a table with gene-name, start and end coordinates
+df_genes = pd.read_csv(
+    "chr22_gene_bounds.csv", sep="\t", names=["gene", "start", "end", "L"]
+).set_index("gene")
+
+# keep track of the few dozen reads that Nikos has selected
+selected_reads = {}
+for fa_id, seq, qual in read_fq("reads_chr22_R2.fastq.gz"):
+    if "IGLC3" in fa_id:
+        print(f"selecting IGLC3 read {fa_id} -> {fa_id.split('_')[0]}")
+    selected_reads[fa_id.split("_")[0]] = fa_id
+
+# then go through the SAM file (no header) to get the mapping position of these reads (not ideal)
+df = pd.read_csv(
+    "/data/rajewsky/home/nkarais/murphy/fc_sts/collect_reads_chr22/final.polyA_adapter_trimmed_chr22.sam",
+    sep="\t",
+    header=None,
+)
+
+starts = []
+for row in df.itertuples():
+    # print(row)
+    qname = row[1]
+    if "A00643:496:HFJ5MDRX2:1:2101:12888:1172" in qname:
+        print(f"YAY! detected IGLC3 read {qname}")
+
+    if qname in selected_reads:
+        print(f"selecting read {qname}")
+        starts.append(row[4])
+
+# find genes that overlap the selected reads mapping position
+# this intersection code is very crude, but effective
+intervals = set()
+starts = set(starts)
+for row in df_genes.itertuples():
+    next_starts = set(starts)
+    for x in starts:
+        if row.start < x and row.end > x:
+            intervals.add((row.start, row.end))
+            print(f"selecting gene entry '{row}'")
+            next_starts.discard(x)
+
+    starts = next_starts
+
+print(
+    f"we have the following start coordinates left. selecting buffer regions around these"
+)
+print(starts)
+
+
+def do_merge(s, e, intervals):
+    keep = []
+    for j, (s2, e2) in enumerate(intervals):
+        new = (s2, e2)
+        if s2 <= e and e2 >= e:
+            print(f"overlap on the right, s={s} e={e} s2={s2} e2={e2}")
+            new = (min(s2, s), max(e, e2))
+        elif e2 >= s and s2 <= s:
+            print(f"overlap on the left, s={s} e={e} s2={s2} e2={e2}")
+            new = (min(s2, s), max(e, e2))
+        elif s2 >= s and e2 <= e:
+            print(f"contained in other interval. discard, s={s} e={e} s2={s2} e2={e2}")
+            continue
+
+        keep.append(new)
+
+    return keep
+
+
+# merge intervals that have some overlap
+intervals = sorted(list(intervals), key=lambda x: x[1] - x[0], reverse=True)
+print(intervals)
+
+while True:
+    changed = False
+    for i, (s, e) in enumerate(intervals):
+        others = intervals[i + 1 :]
+        keep = do_merge(s, e, others)
+        if keep != others:
+            print("we had a change!")
+            print("before")
+            for s, e in intervals:
+                print(f"{s} - {e}")
+
+            intervals = intervals[: i + 1] + keep
+            intervals = sorted(list(intervals), key=lambda x: x[1] - x[0], reverse=True)
+            print("after")
+            for s, e in intervals:
+                print(f"{s} - {e}")
+
+            changed = True
+            break  # the for loop
+
+    if not changed:
+        break
+
+intervals = sorted(list(set(intervals)), key=lambda x: x[1] - x[0], reverse=True)
+print("remaining intervals")
+for s, e in intervals:
+    print(f"{s} - {e}")
+
+
+# Okay, now we know the gene loci which are needed to map the test reads!
+intervals = list(sorted(intervals))
+print(f"relevant intervals found: {len(intervals)}")
+
+# extract the genomic sequence for the loci we need and save as a mini-"genome"
+with open("test_genome.fa", "wt") as f:
+    for start, end in intervals:
+        seq = genome.get_oriented("chr22", start, end, "+")
+        f.write(f">test_chr22.{start}-{end}\n{seq}\n")
+
+# cut down the GTF annotation to only those parts that pertain to the genic regions we care about
+with open("test_annotation.gtf", "wt") as f:
+    for line in open("gencode.v38.chr22.gtf", "rt"):
+        if line.startswith("#"):
+            continue
+
+        parts = line.rstrip().split("\t")
+        chrom, source, rec, start, end = parts[:5]
+        if chrom != "chr22":
+            continue
+
+        start = int(start)
+        end = int(end)
+
+        for s, e in intervals:
+            if (
+                (start <= s and end > s)  # overlap the start of interval
+                or (start > s and end < e)  # internal to interval
+                or (start < e and end > e)  # overlap the end of interval
+                or (start < s and end > e)  # overlap the entire interval
+            ):
+                # the name of the pseudo-chromosome this is on (see excision of genomic sequence above)
+                chrom = f"test_{chrom}.{s}-{e}"
+
+                start = max(
+                    0, start - s
+                )  # translate start and end coordinates from whole chr22 to the gene region
+                end = min(e - s, end - s)
+
+                parts[0:5] = (chrom, "test_data", rec, str(start), str(end))
+                f.write("\t".join(parts) + "\n")
+
+# Done! We now have:
+# * test_genome.fa.gz with the gene sequences
+# * test_annotation.gtf.gz with the gene models (exon/intron, CDS/UTR etc.)
+# * reads_chr22_R1.fastq.gz with test read barcodes mapping to a few CBs and UMIs
+# * reads_chr22_R2.fastq.gz with test read cDNAs mapping to the genic regions we care about
diff --git a/test_data/reads_chr22_R1.fastq.gz b/test_data/reads_chr22_R1.fastq.gz
new file mode 100644
index 00000000..c73ee20a
Binary files /dev/null and b/test_data/reads_chr22_R1.fastq.gz differ
diff --git a/test_data/reads_chr22_R2.fastq.gz b/test_data/reads_chr22_R2.fastq.gz
new file mode 100644
index 00000000..c2c18ddd
Binary files /dev/null and b/test_data/reads_chr22_R2.fastq.gz differ
diff --git a/test_data/test_annotation.gtf.gz b/test_data/test_annotation.gtf.gz
new file mode 100644
index 00000000..1e35c6a2
Binary files /dev/null and b/test_data/test_annotation.gtf.gz differ
diff --git a/test_data/test_config.yaml b/test_data/test_config.yaml
new file mode 100644
index 00000000..6c81ad70
--- /dev/null
+++ b/test_data/test_config.yaml
@@ -0,0 +1,265 @@
+root_dir: '.'
+temp_dir: './tmp'
+external_bin: 
+        dropseq_tools: './bin/dropseq'
+logging:
+  level: INFO
+  debug: "spacemake.util.read_fq"
+
+puck_data:
+    barcode_file: 'predictions_ml.csv'
+    root: 'puck_data'
+
+pucks:
+    default:
+        width_um: 3000
+        spot_diameter_um: 10
+    visium:
+        barcodes: 'puck_data/visium_barcode_positions.csv'
+        width_um: 6500
+        spot_diameter_um: 55
+    seq_scope:
+        width_um: 1000
+        spot_diameter_um: 1
+    slide_seq:
+        width_um: 3000
+        spot_diameter_um: 10
+    test_puck:
+        width_um: 4000
+        spot_diameter_um: 1 
+    openst:
+        width_um: 1200
+        spot_diameter_um: 0.6
+        coordinate_system: 'puck_data/openst_coordinate_system.csv'
+
+run_modes:
+    default:
+        n_beads: 100000
+        umi_cutoff: [100, 300, 500]
+        clean_dge: False
+        detect_tissue: False
+        count_intronic_reads: True
+        count_mm_reads: False
+        mesh_data: False
+        mesh_type: 'circle'
+        mesh_spot_diameter_um: 55
+        mesh_spot_distance_um: 100
+    visium:
+        n_beads: 10000
+        umi_cutoff: [1000]
+        clean_dge: False
+        detect_tissue: True
+        count_intronic_reads: False
+        count_mm_reads: True
+    slide_seq:
+        n_beads: 100000
+        umi_cutoff: [50]
+        clean_dge: False
+        detect_tissue: False
+    scRNA_seq:
+        n_beads: 10000
+        umi_cutoff: [500]
+        detect_tissue: False
+        count_intronic_reads: True
+        count_mm_reads: False
+    seq_scope:
+        clean_dge: false
+        count_intronic_reads: false
+        count_mm_reads: false
+        detect_tissue: false
+        mesh_data: true
+        mesh_spot_diameter_um: 10
+        mesh_spot_distance_um: 15
+        mesh_type: hexagon
+        n_beads: 1000
+        umi_cutoff:
+        - 100
+        - 300
+    spatial_rm:
+        clean_dge: false
+        count_intronic_reads: false
+        count_mm_reads: false
+        detect_tissue: false
+        mesh_data: true
+        mesh_spot_diameter_um: 10
+        mesh_spot_distance_um: 15
+        mesh_type: hexagon
+        n_beads: 1000
+        umi_cutoff:
+        - 500
+        - 1000
+    openst:
+        clean_dge: false
+        count_intronic_reads: true
+        count_mm_reads: true
+        detect_tissue: false
+        mesh_data: true
+        mesh_spot_diameter_um: 7
+        mesh_spot_distance_um: 7
+        mesh_type: hexagon
+        n_beads: 100000
+        polyA_adapter_trimming: true
+        spatial_barcode_min_matches: 0.1
+        umi_cutoff:
+        - 100
+        - 250
+        - 500
+  
+
+barcode_flavors:
+    default:
+        cell: "r1[0:12]"
+        UMI: "r1[12:20]"
+    dropseq:
+        cell: "r1[0:12]"
+        UMI: "r1[12:20]"
+    slide_seq_14bc:
+        cell: "r1[0:14]"
+        UMI: "r1[14:23]"
+    slide_seq_15bc:
+        cell: "r1[0:14]"
+        UMI: "r1[15:23]"
+    visium:
+        cell: "r1[0:16]"
+        UMI: "r1[16:28]"
+    sc_10x_v2:
+        cell: "r1[0:16]"
+        UMI: "r1[16:26]"
+    seq_scope:
+        UMI: "r2[0:9]"
+        cell: "r1[0:20]"
+    nextflex:
+        min_qual_trim: 20
+        cell: "'A'"
+        read1: "None"
+        UMI: "r2[:4] + r2[-4:]"
+        seq: "r2[4:-4]"
+        qual: "r2_qual[4:-4]"
+    openst:
+        UMI: "r2[0:9]"
+        bam_tags: "CR:{cell},CB:{cell},MI:{UMI},RG:{assigned}"
+        cell: "r1[2:27]"
+
+adapters:
+    optical_primer: GAATCACGATACGTACACCA
+    TSO_SMART: AAGCAGTGGTATCAACGCAGAGTGAATGGG
+    SMART: AAGCAGTGGTATCAACGCAGAGTG
+    smart: AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTC
+    TSO_10x: AAGCAGTGGTATCAACGCAGAGTACATGGG
+    chromium_bead: CTACACGACGCTCTTCCGATCT
+    dropseq_bead: AAGCAGTGGTATCAACGCAGAGTAC
+    polyA: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+    polyG: GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG
+    nextflex_RA3: TGGAATTCTCGGGTGCCAAGGAACTCCAGTCACTGACCAATCTCGTATGCCGTCTTCTGCTTGAA
+    truseq_RA3: TGGAATTCTCGGGTGCCAAGGAACTCCAGTCACCCGTCCA
+
+adapter_flavors:
+  SMARTER:
+    cut_right:
+    - Q:
+        min_base_quality: 30
+    - polyA:
+        max_error: 0.25
+        min_overlap: 3
+    paired_end: replace_N
+  chromium:
+    cut_right:
+    - Q:
+        min_base_quality: 32
+    - polyA:
+        max_error: 0.25
+        min_overlap: 3
+    - polyG:
+        max_error: 0.1
+        min_overlap: 3
+  dropseq:
+    cut_left:
+    - TSO_SMART:
+        max_error: 0.1
+        min_overlap: 10
+    cut_right:
+    - Q:
+        min_base_quality: 30
+    - polyA:
+        max_error: 0.25
+        min_overlap: 3
+    - polyG:
+        max_error: 0.1
+        min_overlap: 3
+    paired_end: single-end
+  default:
+    cut_left:
+    - TSO_SMART:
+          max_error: 0.1
+          min_overlap: 10
+    cut_right:
+    - Q:
+        min_base_quality: 30
+    - polyA:
+        max_error: 0.25
+        min_overlap: 3
+    - polyG:
+        max_error: 0.1
+        min_overlap: 3
+    paired_end: single-end
+        
+quant:
+  default:
+    counter_class: "spacemake.quant.DefaultCounter"
+    channels: 
+      - "counts"
+      - "exonic_counts"
+      - "exonic_reads"
+      - "intronic_counts"
+      - "intronic_reads"
+    X_counts: ["exonic_counts", "intronic_counts"]
+    alignment_priorities: {
+        'C': 101, # coding exon
+        'c': 100, # coding exon (lower case == antisense)
+        'U': 51,  # UTR exon
+        'u': 50,
+        'CU': 51, # overlaps both, CDS+UTR (should in fact never occur as 'CU')
+        'cu': 50,
+        'N': 21, # exon of non-coding transcript
+        'n': 20,
+        'I': 11, # intronic region
+        'i': 10,
+        '-': 0,
+    }
+    gene_priorities: {
+        'C': 101, # coding exon
+        'c': 100, # coding exon (lower case == antisense)
+        'U': 51,  # UTR exon
+        'u': 50,
+        'CU': 51, # overlaps both, CDS+UTR (should in fact never occur as 'CU')
+        'cu': 50,
+        'N': 21, # exon of non-coding transcript
+        'n': 20,
+        'I': 11, # intronic region
+        'i': 10,
+        '-': 0,
+    }
+    exonic_tags: ["C", "U", "CU", "N", "c", "u", "cu", "n"]
+    intronic_tags: ["I", "i"]
+    alignment_selection: priority
+    exon_intron_disambiguation: "exon_wins"
+  miRNA:
+    alignment_selection: take_first
+  chrom:
+    alignment_selection: take_first    
+    gene_selection: chrom
+  custom_index:
+    alignment_selection: take_first_plus
+    gene_selection: chrom
+species:
+    test_hsa:
+        genome:
+            annotation: "{spacemake_dir}/test_data/test_annotation.gtf.gz"
+            sequence: "{spacemake_dir}/test_data/test_genome.fa.gz"
+        miRNA:
+            annotation: "{spacemake_dir}/test_data/mirgenedb.hsa.mature.gtf.gz"
+            sequence: "{spacemake_dir}/test_data/mirgenedb.hsa.mature.fa.gz"
+        rRNA:
+            annotation: ''
+            sequence: "{spacemake_dir}/test_data/rRNA_hsa.fa.gz"
+
diff --git a/test_data/test_project_df.csv b/test_data/test_project_df.csv
new file mode 100644
index 00000000..4779bd18
--- /dev/null
+++ b/test_data/test_project_df.csv
@@ -0,0 +1,8 @@
+project_id,sample_id,puck_barcode_file_id,sample_sheet,species,demux_barcode_mismatch,demux_dir,basecalls_dir,R1,R2,reads,longreads,longread_signature,investigator,sequencing_date,experiment,puck_barcode_file,run_mode,barcode_flavor,is_merged,merged_from,puck,dge,map_strategy,adapter_flavor
+test,test_01,['no_spatial_data'],,test_hsa,1,,,['{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz'],['{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz'],,,,unknown,unknown,unknown,,['default'],dropseq,False,[],default,,STAR:genome,dropseq
+test,test_01b,['no_spatial_data'],,test_hsa,1,,,['{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz'],['{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz'],,,,unknown,unknown,unknown,,['default'],dropseq,False,[],default,,STAR:genome,dropseq
+test,test_02,['no_spatial_data'],,test_hsa,1,,,['{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz'],['{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz'],,,,unknown,unknown,unknown,,['default'],dropseq,False,[],default,,rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final,dropseq
+test,test_03_nofinal,['no_spatial_data'],,test_hsa,1,,,['{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz'],['{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz'],,,,unknown,unknown,unknown,,['default'],dropseq,False,[],default,,rRNA:bowtie2->miRNA:bowtie2->genome:STAR,dropseq
+tile,tile_1,['tile_1'],,test_hsa,1,,,['{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz'],['{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz'],,,,unknown,unknown,unknown,['{spacemake_dir}/test_data/tile_1.txt'],['spatial_rm'],dropseq,False,[],test_puck,,rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final,dropseq
+tile,tile_2,['tile_2'],,test_hsa,1,,,['{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz'],['{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz'],,,,unknown,unknown,unknown,['{spacemake_dir}/test_data/tile_2.txt'],['spatial_rm'],dropseq,False,[],test_puck,,rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final,dropseq
+tile,tile_both,"['tile_1', 'tile_2']",,test_hsa,1,,,['{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz'],['{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz'],,,,unknown,unknown,unknown,"['{spacemake_dir}/test_data/tile_1.txt', '{spacemake_dir}/test_data/tile_2.txt']",['spatial_rm'],dropseq,False,[],test_puck,,rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final,dropseq
diff --git a/tests/fixtures.py b/tests/fixtures.py
new file mode 100644
index 00000000..9b50e6de
--- /dev/null
+++ b/tests/fixtures.py
@@ -0,0 +1,142 @@
+import pytest
+import os
+
+spacemake_dir = os.path.abspath(os.path.dirname(__file__) + '/../')
+print("SPACEMAKE_DIR", spacemake_dir)
+
+def sm(*argc, expect_fail=False):
+    # construct the desired cmdline
+    import sys
+    sys.argv = ["spacemake",] + list(argc)
+    
+    # ensure that no ConfigFile and ProjectDF instances 
+    # are retained from previous tests
+    import spacemake.config
+    import spacemake.project_df
+    spacemake.config.__global_config = None
+    spacemake.project_df.__global_ProjectDF = None
+
+    # execute spacemake cmdline code
+    from spacemake.cmdline import cmdline
+    res = cmdline()
+    # print("res", res)
+    if expect_fail:
+        assert isinstance(res, Exception) == True
+    else:
+        assert isinstance(res, Exception) == False
+
+    return res
+
+def _init():
+    # just get the version
+    sm("--version")
+
+    # test the init parser
+    sm("init", "--dropseq_tools", "/data/rajewsky/shared_bins/Drop-seq_tools-2.5.1/")
+
+
+def _add_species():
+    sm(
+        "config", "add_species",
+        "--name=test_hsa",
+        "--reference=genome",
+        f"--sequence={spacemake_dir}/test_data/test_genome.fa.gz",
+        f"--annotation={spacemake_dir}/test_data/test_annotation.gtf.gz",
+    )
+    # add a second reference
+    sm(
+        "config", "add_species",
+        "--name=test_hsa",
+        "--reference=rRNA",
+        f"--sequence={spacemake_dir}/test_data/rRNA_hsa.fa.gz",
+    )
+    # add a third reference
+    sm(
+        "config", "add_species",
+        "--name=test_hsa",
+        "--reference=miRNA",
+        f"--sequence={spacemake_dir}/test_data/mirgenedb.hsa.mature.fa.gz",
+    )
+    # pretend we have mouse as well
+    # TODO: place some actual mouse genome and/or phiX genomes in test-data repository
+    sm(
+        "config", "add_species",
+        "--name=mouse",
+        "--reference=genome",
+        f"--sequence={spacemake_dir}/test_data/test_genome.fa.gz",
+        f"--annotation={spacemake_dir}/test_data/test_annotation.gtf.gz",
+    )
+    sm(
+        "config", "add_species",
+        "--name=mouse",
+        "--reference=phiX",
+        f"--sequence={spacemake_dir}/test_data/test_genome.fa.gz",
+        f"--annotation={spacemake_dir}/test_data/test_annotation.gtf.gz",
+    )
+    sm(
+        "config", "add_species",
+        "--name=mouse",
+        "--reference=rRNA",
+        f"--sequence={spacemake_dir}/test_data/rRNA_hsa.fa.gz",
+    )
+
+
+@pytest.fixture
+def tmp_root(tmp_path_factory):
+    tmp = tmp_path_factory.mktemp("root_blank")
+
+    return tmp
+
+
+@pytest.fixture
+def initialized_root(tmp_path_factory):
+    tmp = tmp_path_factory.mktemp("root_initialized")
+    os.chdir(tmp.as_posix())
+
+    _init()
+    return tmp
+
+
+@pytest.fixture
+def with_species(initialized_root):
+    os.chdir(initialized_root.as_posix())
+    # # test old way
+    # sm(
+    #     "config", "add_species",
+    #     "--name=hsa_test",
+    #     f"--genome={spacemake_dir}/test_data/test_genome.fa.gz",
+    #     f"--annotation={spacemake_dir}/test_data/test_annotation.gtf.gz",
+    # )
+    # test new way
+    _add_species()
+    return initialized_root
+
+
+@pytest.fixture
+def configured_root(tmp_path_factory):
+    tmp_root = tmp_path_factory.mktemp("root_preconfigured")
+
+    sm_path = os.path.dirname(__file__)
+    # make a tmp-copy of the test_config.yaml
+    def_config = os.path.join(sm_path, "../test_data/test_config.yaml")
+    os.system(f"cp {def_config} {tmp_root / 'config.yaml'}")
+
+    test_pdf =  os.path.join(sm_path, "../test_data/test_project_df.csv")
+    open(f"{tmp_root / 'project_df.csv'}", 'w').write(
+        open(test_pdf, 'r').read().format(spacemake_dir=spacemake_dir)
+    )
+    #os.system(f"cp {test_pdf} {tmp_root / 'project_df.csv'}")
+
+    return tmp_root
+
+
+@pytest.fixture(scope="session")
+def with_tile_test_data(tmp_path_factory):
+    tmp = tmp_path_factory.mktemp("root_tile_test")
+    os.chdir(tmp.as_posix())
+    _init()
+    _add_species()
+    print("return code", os.system("wget https://bimsbstatic.mdc-berlin.de/rajewsky/spacemake-test-data/spacemake_tile_test_data.tar.gz -O /dev/stdout | tar -xz"))
+    print(os.listdir('.'))
+
+    return tmp
\ No newline at end of file
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
new file mode 100644
index 00000000..b825abc6
--- /dev/null
+++ b/tests/test_cmdline.py
@@ -0,0 +1,332 @@
+import pytest
+import sys
+import os
+from spacemake.cmdline import *
+
+from fixtures import initialized_root, with_species, with_tile_test_data, configured_root, sm, spacemake_dir
+
+
+def test_init(initialized_root):
+    assert os.path.exists(initialized_root.as_posix() + '/config.yaml')
+
+
+def test_parsers(configured_root):
+    get_project_sample_parser()
+    get_add_sample_sheet_parser()
+    
+    parser = get_sample_main_variables_parser()
+    get_action_sample_parser(parser.add_subparsers(), "add", lambda *argc, **kw: None)
+
+    get_sample_extra_info_parser()
+    get_data_parser()
+    get_run_parser()
+
+    make_main_parser()
+
+
+def test_subcommands(configured_root):
+    # test everything after init
+    os.chdir(configured_root.as_posix())
+
+    sm("projects", "list")
+    sm("config", "list_adapter_flavors")
+
+
+def test_config_barcodes(initialized_root):
+    os.chdir(initialized_root.as_posix())
+
+    # add
+    sm(
+        "config", "add_barcode_flavor",
+        "--name", "fc_sts_miniseq",
+        "--umi", "r2[0:9]", "--cell_barcode", "r1[2:27]"
+    )
+    # add same bc flavor twice? -> error
+    sm(
+        "config", "add_barcode_flavor",
+        "--name", "fc_sts_miniseq",
+        "--umi", "r2[0:9]", "--cell_barcode", "r1[2:27]",
+        expect_fail=True
+    )
+    # delete
+    sm(
+        "config", "delete_barcode_flavor",
+        "--name", "fc_sts_miniseq",
+    )
+    # re-add
+    sm(
+        "config", "add_barcode_flavor",
+        "--name", "fc_sts_miniseq",
+        "--umi", "r2[0:8]", "--cell_barcode", "r1[2:27]"
+    )
+    # update
+    sm(
+        "config", "update_barcode_flavor",
+        "--name", "fc_sts_miniseq",
+        "--umi", "r2[0:9]"
+    )
+
+
+def test_config_adapters(initialized_root):
+    os.chdir(initialized_root.as_posix())
+    # add
+    sm(
+        "config", "add_adapter",
+        "--name", "testy", "--seq", "ACGTACGTACGTACGT",
+    )
+    sm(
+        "config", "add_adapter",
+        "--name", "testy", "--seq", "ACGTACGTACGTACGT",
+        expect_fail=True
+    )
+    # update
+    sm(
+        "config", "update_adapter",
+        "--name", "testy", "--seq", "ACGTACGTACGTACGTA",
+    )
+    # delete
+    sm(
+        "config", "delete_adapter",
+        "--name", "testy"
+    )
+    # re-add
+    sm(
+        "config", "add_adapter",
+        "--name", "testy", "--seq", "ACGTACGTACGTACGTACGT",
+    )
+
+
+def test_config_adapter_flavors(initialized_root):
+    os.chdir(initialized_root.as_posix())
+
+    test_flavor = (
+        "--name", "testy1",
+        "--cut_left", 
+        "SMART:min_overlap=10:max_errors=0.1",
+        "--cut_right", 
+        "Q:min_base_qual=30", 
+        "polyG:min_overlap=3:max_errors=0.2",
+        "polyA:min_overlap=3:max_errors=0.25"
+    )
+
+    # add
+    sm("config", "add_adapter_flavor", *test_flavor)
+    sm("config", "add_adapter_flavor", *test_flavor, expect_fail=True)
+    # update -> not implemented
+    sm("config", "update_adapter_flavor", "--name", "testy1", expect_fail=True)
+    # delete
+    sm("config", "delete_adapter_flavor", "--name", "testy1")
+    # re-add
+    sm("config", "add_adapter_flavor", *test_flavor)
+
+
+def test_species(with_species):
+    pass
+
+
+def test_puck(initialized_root):
+    os.chdir(initialized_root.as_posix())
+    test_puck = (
+        "--name=test_puck",
+        "--width_um=2",
+        "--spot_diameter_um=0.5",
+    )
+    # add
+    sm("config", "add_puck", *test_puck)
+    sm("config", "add_puck", *test_puck, expect_fail=True)
+    # update
+    sm("config", "update_puck", "--name=test_puck", "--width_um=3")
+    # delete
+    sm("config", "delete_puck", "--name=test_puck")
+    # re-add
+    sm("config", "add_puck", *test_puck)
+
+
+def test_runmode(initialized_root):
+    os.chdir(initialized_root.as_posix())
+    # add
+    sm("config", "add_run_mode", "--name=spatial_rm", "--umi_cutoff=10")
+    sm("config", "add_run_mode", "--name=spatial_rm2","--umi_cutoff=10")
+    # delete
+    sm("config", "delete_run_mode", "--name=spatial_rm2")
+    # edit
+    sm("config", "update_run_mode", "--name=spatial_rm", "--umi_cutoff=1")
+    from spacemake.config import get_global_config
+    config = get_global_config()
+    config.dump()
+
+def test_issue_88(initialized_root):
+    os.chdir(initialized_root.as_posix())
+    # name added to config.yaml when creating a new run mode #88
+    # add
+    sm("config", "add_run_mode", "--name", "newrunmode")
+    from spacemake.config import get_global_config
+    config = get_global_config()
+    config.dump()
+    inserted = config.variables['run_modes']['newrunmode']
+    print("here's what we inserted", inserted)
+    assert "name" not in inserted
+
+
+def test_sample(configured_root):
+    os.chdir(configured_root.as_posix())
+    test_sample = (
+        f'--R1={spacemake_dir}/test_data/reads_chr22_R1.fastq.gz', 
+        f'--R2={spacemake_dir}/test_data/reads_chr22_R1.fastq.gz', 
+        '--map_strategy=genome:STAR:final',
+        '--species=test_hsa'
+    )
+
+    # add
+    sm(
+        "projects", "add_sample",
+        "--project_id=test", "--sample_id=test1", 
+        *test_sample
+    )
+    # delete
+    sm(
+        "projects", "delete_sample",
+        "--project_id=test", "--sample_id=test1", 
+    )
+    # re-add
+    sm(
+        "projects", "add_sample",
+        "--project_id=test", "--sample_id=test1", 
+        *test_sample
+    )
+    # update
+    sm(
+        "projects", "update_sample",
+        "--project_id=test", "--sample_id=test1",
+        '--map_strategy=rRNA:bowtie2->genome:STAR'
+    )
+
+def test_fill_project_df(with_species):
+    os.chdir(with_species.as_posix())
+    test_project_data = [
+        (
+            "test_hsa",
+            "test",
+            "test_01",
+            f"{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz",
+            f"{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz",
+            "--map_strategy=genome:STAR:final",
+        ),
+        (
+            "test_hsa",
+            "test",
+            "test_02.2",
+            f"{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz",
+            f"{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz",
+            "--map_strategy=rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final",
+        ),
+        (
+            "test_hsa",
+            "test",
+            "test_03_nofinal",
+            f"{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz",
+            f"{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz",
+            "--map_strategy=rRNA:bowtie2->miRNA:bowtie2->genome:STAR",
+        ),
+        (
+            "test_hsa",
+            "test",
+            "test_bulk",
+            "None",
+            f"{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz",
+            "--map_strategy=rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final"
+            " --barcode_flavor=visium",
+        ),
+        (
+            "test_hsa",
+            "tile",
+            "tile_1",
+            f"{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz",
+            f"{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz",
+            (
+                "--map_strategy=rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final"
+                f" --puck_barcode_file {spacemake_dir}/test_data/tile_1.txt"
+                " --puck slide_seq --run_mode slide_seq"
+            )
+        ),
+        (
+            "test_hsa",
+            "tile",
+            "tile_2",
+            f"{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz",
+            f"{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz",
+            (
+                "--map_strategy=rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final"
+                f" --puck_barcode_file {spacemake_dir}/test_data/tile_2.txt"
+                " --puck slide_seq --run_mode slide_seq"
+            )
+        ),
+        (
+            "test_hsa",
+            "tile",
+            "tile_both",
+            f"{spacemake_dir}/test_data/reads_chr22_R1.fastq.gz",
+            f"{spacemake_dir}/test_data/reads_chr22_R2.fastq.gz",
+            (
+                "--map_strategy=rRNA:bowtie2->miRNA:bowtie2->genome:STAR:final"
+                f" --puck_barcode_file {spacemake_dir}/test_data/tile_1.txt {spacemake_dir}/test_data/tile_2.txt"
+                " --puck slide_seq --run_mode slide_seq"
+            )
+        ),
+    ]
+    for species, project_id, sample_id, R1, R2, extra in test_project_data:
+        # add
+        sm(
+            "projects", "add_sample",
+            f"--project_id={project_id}", f"--sample_id={sample_id}",
+            f"--species={species}",
+            f"--R1={R1}", f"--R2={R2}", *extra.split(' ')
+        )
+
+def test_run(configured_root):
+    # test everything after init
+    os.chdir(configured_root.as_posix())
+    sm("run", "-np", "--cores=8")
+
+@pytest.mark.big_download
+def test_tiles(with_tile_test_data):
+    os.chdir(with_tile_test_data.as_posix())
+    sm(
+        "config", "add_puck",
+        "--name=openst_fc_010",
+        "--coordinate_system=fc_010_coordinate_system.csv",
+        "--width_um=1200",
+        "--spot_diameter_um=0.6"
+    )
+    sm(
+        "projects", "add_sample",
+        "--project_id=fc_sts_75",
+        "--sample_id=fc_sts_75_1b",
+        "--R1=fc_sts_75_1b_S1_R1_001_subsampled.fastq.gz",
+        "--R2=fc_sts_75_1b_S1_R2_001_subsampled.fastq.gz",
+        "--species=mouse",
+        "--puck=openst_fc_010",
+        "--run_mode=openst",
+        "--barcode_flavor=openst",
+        "--puck_barcode_file",
+        "fc_010_L2_tile_2558.txt.gz",
+        "fc_010_L2_tile_2559.txt.gz",
+        "fc_010_L4_tile_2505.txt.gz",
+        '--map_strategy=phiX:bowtie2->rRNA:bowtie2->genome:STAR'
+    )
+
+def test_merge(configured_root):
+    os.chdir(configured_root.as_posix())
+    sm(
+        "projects", "merge_samples",
+        "--merged_project_id=test",
+        "--merged_sample_id=test_merged",
+        "--sample_id_list", "test_01", "test_02",
+        expect_fail=True # different map_strategies
+    )
+    sm(
+        "projects", "merge_samples",
+        "--merged_project_id=test",
+        "--merged_sample_id=test_merged",
+        "--sample_id_list", "test_01", "test_01b",
+    )    
\ No newline at end of file
diff --git a/tests/test_fastq_to_ubam.py b/tests/test_fastq_to_ubam.py
new file mode 100644
index 00000000..6d17591d
--- /dev/null
+++ b/tests/test_fastq_to_ubam.py
@@ -0,0 +1,77 @@
+import multiprocessing as mp
+
+import pytest
+import sys
+import os
+from spacemake.bin.fastq_to_uBAM import *
+
+
+spacemake_dir = os.path.dirname(__file__) + "/../"
+
+
+@pytest.fixture(scope="session")
+def test_root(tmp_path_factory):
+    tmp = tmp_path_factory.mktemp("root")
+    sm_path = os.path.dirname(__file__)
+    # make a tmp-copy of the test_config.yaml
+    # def_config = os.path.join(sm_path, "../test_data/test_config.yaml")
+    # os.system(f"cp {def_config} {tmp / 'config.yaml'}")
+
+    # test_pdf =  os.path.join(sm_path, "../test_data/test_project_df.csv")
+    # os.system(f"cp {test_pdf} {tmp / 'project_df.csv'}")
+
+    return tmp
+
+
+def sm(*argc, expect_fail=False):
+    sys.argv = [
+        "fastq_to_uBAM.py",
+    ] + list(argc)
+    res = cmdline()
+    print("got result", res)
+    from spacemake.errors import SpacemakeError
+
+    if expect_fail:
+        assert isinstance(res, SpacemakeError) == True
+    else:
+        assert isinstance(res, Exception) == False
+
+
+def test_help():
+    try:
+        sm("--help")
+    except SystemExit:
+        pass
+
+
+def test_dropseq():
+    sm(
+        "--read1",
+        spacemake_dir + "test_data/reads_chr22_R1.fastq.gz",
+        "--read2",
+        spacemake_dir + "test_data/reads_chr22_R2.fastq.gz",
+        "--out-bam",
+        "/dev/null",
+    )
+
+
+def test_single():
+    sm(
+        "--read2",
+        spacemake_dir + "test_data/reads_chr22_R2.fastq.gz",
+        "--out-bam",
+        "/dev/null",
+        """--cell='"A"'""",
+    )
+
+
+def test_minqual():
+    sm(
+        "--read2",
+        spacemake_dir + "test_data/reads_chr22_R2.fastq.gz",
+        "--out-bam",
+        "/dev/null",
+        "--min-qual",
+        "30",
+        """--cell='"A"'""",
+    )
diff --git a/tests/test_map_strategy.py b/tests/test_map_strategy.py
new file mode 100644
index 00000000..465760b4
--- /dev/null
+++ b/tests/test_map_strategy.py
@@ -0,0 +1,72 @@
+import pytest
+
+from spacemake.map_strategy import *
+from spacemake.config import ConfigFile
+from spacemake.project_df import ProjectDF
+from spacemake.errors import *
+import os
+
+from fixtures import configured_root, tmp_root, sm, spacemake_dir
+
+def test_validation(configured_root):
+    config = ConfigFile.from_yaml((configured_root / "config.yaml").as_posix())
+    data = [
+        ("flipped", "bowtie2:rRNA->STAR:genome", 'test_hsa', "rRNA:bowtie2->genome:STAR"),
+        ("species_missing", "bowtie2:rRNA->STAR:genome", 'test_hs', "<class 'spacemake.errors.ConfigVariableNotFoundError'>"),
+        ("with_cflavor", "rRNA:bowtie2@custom_index->genome:STAR@default", 'test_hsa', "rRNA:bowtie2@custom_index->genome:STAR@default"),
+        ("unknown_cflavor", "rRNA:bowtie2@custom->genome:STAR@default", 'test_hsa', "<class 'spacemake.errors.ConfigVariableNotFoundError'>"),
+        # ("flipped", "bowtie2:rRNA->STAR:genome", "rRNA:bowtie2->genome:STAR"),
+    ]
+    for name, mapstr, species, expect in data:
+        # print(f"running test {name}")
+        try:
+            res = validate_mapstr(mapstr, config=config, species=species)
+        except (ValueError, ConfigVariableNotFoundError) as e:
+            res = str(type(e))
+
+        print(f"test '{name}': {mapstr}-> {res} expect={expect} {expect == res}")
+        assert res == expect
+
+
+def test_mapstr(configured_root):
+    config = ConfigFile.from_yaml((configured_root / "config.yaml").as_posix())
+    data = [
+        ("with_cflavor", "rRNA:bowtie2@custom_index->genome:STAR@default", None),
+    ]
+    for name, mapstr, expect in data:
+        mr, lr = mapstr_to_targets(mapstr)
+        assert mr[0].input_name == "uBAM"
+        assert mr[-1].input_name == "rRNA.bowtie2"
+        assert lr[0].link_src == "genome.STAR"
+        assert lr[0].link_name == "final"
+    
+
+def test_get_mapped_BAM_output(configured_root):
+    config = ConfigFile.from_yaml((configured_root / "config.yaml").as_posix())
+    project_df = ProjectDF((configured_root / "project_df.csv").as_posix(), config=config)
+
+    out_files = get_mapped_BAM_output(project_df=project_df, config=config)
+    print(out_files)
+
+
+def test_validation_cmdline_issue_54(configured_root):
+    os.chdir(configured_root.as_posix())
+    data = [
+        ("flipped", "bowtie2:rRNA->STAR:genome", 'test_hsa', True),
+        ("species_missing", "bowtie2:rRNA->STAR:genome", 'test_hs', False),
+        ("with_cflavor", "rRNA:bowtie2@custom_index->genome:STAR@default", 'test_hsa', True),
+        ("unknown_cflavor", "rRNA:bowtie2@customX->genome:STAR@defaultBLA", 'test_hsa', False),
+        # ("flipped", "bowtie2:rRNA->STAR:genome", "rRNA:bowtie2->genome:STAR"),
+    ]
+    for name, mapstr, species, expect_pass in data:
+        print(f"running test {name}")
+        # add
+        sm(
+            "projects", "add_sample",
+            "--project_id=test", f"--sample_id={name}",
+            f"--map_strategy={mapstr}",
+            f'--R1={spacemake_dir}/test_data/reads_chr22_R1.fastq.gz', 
+            f'--R2={spacemake_dir}/test_data/reads_chr22_R1.fastq.gz', 
+            f'--species={species}',
+            expect_fail=not expect_pass
+        )