stardist_pkg/bioimageio_utils.py

from pathlib import Path
from pkg_resources import get_distribution
from zipfile import ZipFile
import numpy as np
import tempfile
from distutils.version import LooseVersion
from csbdeep.utils import axes_check_and_normalize, normalize, _raise


DEEPIMAGEJ_MACRO = \
"""
//*******************************************************************
// Date: July-2021
// Credits: StarDist, DeepImageJ
// URL:
//      https://github.com/stardist/stardist
//      https://deepimagej.github.io/deepimagej
// This macro was adapted from
// https://github.com/deepimagej/imagej-macros/blob/648caa867f6ccb459649d4d3799efa1e2e0c5204/StarDist2D_Post-processing.ijm
// Please cite the respective contributions when using this code.
//*******************************************************************
//  Macro to run StarDist postprocessing on 2D images.
//  StarDist and deepImageJ plugins need to be installed.
//  The macro assumes that the image to process is a stack in which
//  the first channel corresponds to the object probability map
//  and the remaining channels are the radial distances from each
//  pixel to the object boundary.
//*******************************************************************

// Get the name of the image to call it
getDimensions(width, height, channels, slices, frames);
name=getTitle();

probThresh={probThresh};
nmsThresh={nmsThresh};

// Isolate the detection probability scores
run("Make Substack...", "channels=1");
rename("scores");

// Isolate the oriented distances
run("Fire");
selectWindow(name);
run("Delete Slice", "delete=channel");
selectWindow(name);
run("Properties...", "channels=" + maxOf(channels, slices) - 1 + " slices=1 frames=1 pixel_width=1.0000 pixel_height=1.0000 voxel_depth=1.0000");
rename("distances");
run("royal");

// Run StarDist plugin
run("Command From Macro", "command=[de.csbdresden.stardist.StarDist2DNMS], args=['prob':'scores', 'dist':'distances', 'probThresh':'" + probThresh + "', 'nmsThresh':'" + nmsThresh + "', 'outputType':'Both', 'excludeBoundary':'2', 'roiPosition':'Stack', 'verbose':'false'], process=[false]");
"""


def _import(error=True):
    try:
        from importlib_metadata import metadata
        from bioimageio.core.build_spec import build_model # type: ignore
        import xarray as xr
        import bioimageio.core # type: ignore
    except ImportError:
        if error:
            raise RuntimeError(
                "Required libraries are missing for bioimage.io model export.\n"
                "Please install StarDist as follows: pip install 'stardist[bioimageio]'\n"
                "(You do not need to uninstall StarDist first.)"
            )
        else:
            return None
    return metadata, build_model, bioimageio.core, xr


def _create_stardist_dependencies(outdir):
    from ruamel.yaml import YAML
    from tensorflow import __version__ as tf_version
    from . import __version__ as stardist_version
    pkg_info = get_distribution("stardist")
    # dependencies that start with the name "bioimageio" will be added as conda dependencies
    reqs_conda = [str(req) for req in pkg_info.requires(extras=['bioimageio']) if str(req).startswith('bioimageio')]
    # only stardist and tensorflow as pip dependencies
    tf_major, tf_minor = LooseVersion(tf_version).version[:2]
    reqs_pip = (f"stardist>={stardist_version}", f"tensorflow>={tf_major}.{tf_minor},<{tf_major+1}")
    # conda environment
    env = dict(
        name = 'stardist',
        channels = ['defaults', 'conda-forge'],
        dependencies = [
            ('python>=3.7,<3.8' if tf_major == 1 else 'python>=3.7'),
            *reqs_conda,
            'pip', {'pip': reqs_pip},
        ],
    )
    yaml = YAML(typ='safe')
    path = outdir / "environment.yaml"
    with open(path, "w") as f:
        yaml.dump(env, f)
    return f"conda:{path}"


def _create_stardist_doc(outdir):
    doc_path = outdir / "README.md"
    text = (
        "# StarDist Model\n"
        "This is a model for object detection with star-convex shapes.\n"
        "Please see the [StarDist repository](https://github.com/stardist/stardist) for details."
    )
    with open(doc_path, "w") as f:
        f.write(text)
    return doc_path


def _get_stardist_metadata(outdir, model):
    metadata, *_ = _import()
    package_data = metadata("stardist")
    doi_2d = "https://doi.org/10.1007/978-3-030-00934-2_30"
    doi_3d = "https://doi.org/10.1109/WACV45572.2020.9093435"
    authors = {
        'Martin Weigert': dict(name='Martin Weigert', github_user='maweigert'),
        'Uwe Schmidt': dict(name='Uwe Schmidt', github_user='uschmidt83'),
    }
    data = dict(
        description=package_data["Summary"],
        authors=list(authors.get(name.strip(),dict(name=name.strip())) for name in package_data["Author"].split(",")),
        git_repo=package_data["Home-Page"],
        license=package_data["License"],
        dependencies=_create_stardist_dependencies(outdir),
        cite=[{"text": "Cell Detection with Star-Convex Polygons", "doi": doi_2d},
              {"text": "Star-convex Polyhedra for 3D Object Detection and Segmentation in Microscopy", "doi": doi_3d}],
        tags=[
            'fluorescence-light-microscopy', 'whole-slide-imaging', 'other', # modality
            f'{model.config.n_dim}d', # dims
            'cells', 'nuclei', # content
            'tensorflow', # framework
            'fiji', # software
            'unet', # network
            'instance-segmentation', 'object-detection', # task
            'stardist',
        ],
        covers=["https://raw.githubusercontent.com/stardist/stardist/master/images/stardist_logo.jpg"],
        documentation=_create_stardist_doc(outdir),
    )
    return data


def _predict_tf(model_path, test_input):
    import tensorflow as tf
    from csbdeep.utils.tf import IS_TF_1
    # need to unzip the model assets
    model_assets = model_path.parent / "tf_model"
    with ZipFile(model_path, "r") as f:
        f.extractall(model_assets)
    if IS_TF_1:
        # make a new graph, i.e. don't use the global default graph
        with tf.Graph().as_default():
            with tf.Session() as sess:
                tf_model = tf.saved_model.load_v2(str(model_assets))
                x = tf.convert_to_tensor(test_input, dtype=tf.float32)
                model = tf_model.signatures["serving_default"]
                y = model(x)
                sess.run(tf.global_variables_initializer())
                output = sess.run(y["output"])
    else:
        tf_model = tf.saved_model.load(str(model_assets))
        x = tf.convert_to_tensor(test_input, dtype=tf.float32)
        model = tf_model.signatures["serving_default"]
        y = model(x)
        output = y["output"].numpy()
    return output


def _get_weights_and_model_metadata(outdir, model, test_input, test_input_axes, test_input_norm_axes, mode, min_percentile, max_percentile):

    # get the path to the exported model assets (saved in outdir)
    if mode == "keras_hdf5":
        raise NotImplementedError("Export to keras format is not supported yet")
    elif mode == "tensorflow_saved_model_bundle":
        assets_uri = outdir / "TF_SavedModel.zip"
        model_csbdeep = model.export_TF(assets_uri, single_output=True, upsample_grid=True)
    else:
        raise ValueError(f"Unsupported mode: {mode}")

    # to force "inputs.data_type: float32" in the spec (bonus: disables normalization warning in model._predict_setup)
    test_input = test_input.astype(np.float32)

    # convert test_input to axes_net semantics and shape, also resize if necessary (to adhere to axes_net_div_by)
    test_input, axes_img, axes_net, axes_net_div_by, *_ = model._predict_setup(
        img=test_input,
        axes=test_input_axes,
        normalizer=None,
        n_tiles=None,
        show_tile_progress=False,
        predict_kwargs={},
    )

    # normalization axes string and numeric indices
    axes_norm = set(axes_net).intersection(set(axes_check_and_normalize(test_input_norm_axes, disallowed='S')))
    axes_norm = "".join(a for a in axes_net if a in axes_norm)  # preserve order of axes_net
    axes_norm_num = tuple(axes_net.index(a) for a in axes_norm)

    # normalize input image
    test_input_norm = normalize(test_input, pmin=min_percentile, pmax=max_percentile, axis=axes_norm_num)

    net_axes_in = axes_net.lower()
    net_axes_out = axes_check_and_normalize(model._axes_out).lower()
    ndim_tensor = len(net_axes_out) + 1

    input_min_shape = list(axes_net_div_by)
    input_min_shape[axes_net.index('C')] = model.config.n_channel_in
    input_step = list(axes_net_div_by)
    input_step[axes_net.index('C')] = 0

    # add the batch axis to shape and step
    input_min_shape = [1] + input_min_shape
    input_step = [0] + input_step

    # the axes strings in bioimageio convention
    input_axes = "b" + net_axes_in.lower()
    output_axes = "b" + net_axes_out.lower()

    if mode == "keras_hdf5":
        output_names = ("prob", "dist") + (("class_prob",) if model._is_multiclass() else ())
        output_n_channels = (1, model.config.n_rays,) + ((1,) if model._is_multiclass() else ())
        # the output shape is computed from the input shape using
        # output_shape[i] = output_scale[i] * input_shape[i] + 2 * output_offset[i]
        output_scale = [1]+list(1/g for g in model.config.grid) + [0]
        output_offset = [0]*(ndim_tensor)

    elif mode == "tensorflow_saved_model_bundle":
        if model._is_multiclass():
            raise NotImplementedError("Tensorflow SavedModel not supported for multiclass models yet")
        # regarding input/output names: https://github.com/CSBDeep/CSBDeep/blob/b0d2f5f344ebe65a9b4c3007f4567fe74268c813/csbdeep/utils/tf.py#L193-L194
        input_names = ["input"]
        output_names = ["output"]
        output_n_channels = (1 + model.config.n_rays,)
        # the output shape is computed from the input shape using
        # output_shape[i] = output_scale[i] * input_shape[i] + 2 * output_offset[i]
        # same shape as input except for the channel dimension
        output_scale = [1]*(ndim_tensor)
        output_scale[output_axes.index("c")] = 0
        # no offset, except for the input axes, where it is output channel / 2
        output_offset = [0.0]*(ndim_tensor)
        output_offset[output_axes.index("c")] = output_n_channels[0] / 2.0

    assert all(s in (0, 1) for s in output_scale), "halo computation assumption violated"
    halo = model._axes_tile_overlap(output_axes.replace('b', 's'))
    halo = [int(np.ceil(v/8)*8) for v in halo]  # optional: round up to be divisible by 8

    # the output shape needs to be valid after cropping the halo, so we add the halo to the input min shape
    input_min_shape = [ms + 2 * ha for ms, ha in zip(input_min_shape, halo)]

    # make sure the input min shape is still divisible by the min axis divisor
    input_min_shape = input_min_shape[:1] + [ms + (-ms % div_by) for ms, div_by in zip(input_min_shape[1:], axes_net_div_by)]
    assert all(ms % div_by == 0 for ms, div_by in zip(input_min_shape[1:], axes_net_div_by))

    metadata, *_ = _import()
    package_data = metadata("stardist")
    is_2D = model.config.n_dim == 2

    weights_file = outdir / "stardist_weights.h5"
    model.keras_model.save_weights(str(weights_file))

    config = dict(
        stardist=dict(
            python_version=package_data["Version"],
            thresholds=dict(model.thresholds._asdict()),
            weights=weights_file.name,
            config=vars(model.config),
        )
    )

    if is_2D:
        macro_file = outdir / "stardist_postprocessing.ijm"
        with open(str(macro_file), 'w', encoding='utf-8') as f:
            f.write(DEEPIMAGEJ_MACRO.format(probThresh=model.thresholds.prob, nmsThresh=model.thresholds.nms))
        config['stardist'].update(postprocessing_macro=macro_file.name)

    n_inputs = len(input_names)
    assert n_inputs == 1
    input_config = dict(
        input_names=input_names,
        input_min_shape=[input_min_shape],
        input_step=[input_step],
        input_axes=[input_axes],
        input_data_range=[["-inf", "inf"]],
        preprocessing=[[dict(
            name="scale_range",
            kwargs=dict(
                mode="per_sample",
                axes=axes_norm.lower(),
                min_percentile=min_percentile,
                max_percentile=max_percentile,
            ))]]
        )

    n_outputs = len(output_names)
    output_config = dict(
        output_names=output_names,
        output_data_range=[["-inf", "inf"]] * n_outputs,
        output_axes=[output_axes] * n_outputs,
        output_reference=[input_names[0]] * n_outputs,
        output_scale=[output_scale] * n_outputs,
        output_offset=[output_offset] * n_outputs,
        halo=[halo] * n_outputs
    )

    in_path = outdir / "test_input.npy"
    np.save(in_path, test_input[np.newaxis])

    if mode == "tensorflow_saved_model_bundle":
        test_outputs = _predict_tf(assets_uri, test_input_norm[np.newaxis])
    else:
        test_outputs = model.predict(test_input_norm)

    # out_paths = []
    # for i, out in enumerate(test_outputs):
    #     p = outdir / f"test_output{i}.npy"
    #     np.save(p, out)
    #     out_paths.append(p)
    assert n_outputs == 1
    out_paths = [outdir / "test_output.npy"]
    np.save(out_paths[0], test_outputs)

    from tensorflow import __version__ as tf_version
    data = dict(weight_uri=assets_uri, test_inputs=[in_path], test_outputs=out_paths,
                config=config, tensorflow_version=tf_version)
    data.update(input_config)
    data.update(output_config)
    _files = [str(weights_file)]
    if is_2D:
        _files.append(str(macro_file))
    data.update(attachments=dict(files=_files))

    return data


def export_bioimageio(
    model,
    outpath,
    test_input,
    test_input_axes=None,
    test_input_norm_axes='ZYX',
    name=None,
    mode="tensorflow_saved_model_bundle",
    min_percentile=1.0,
    max_percentile=99.8,
    overwrite_spec_kwargs=None,
):
    """Export stardist model into bioimage.io format, https://github.com/bioimage-io/spec-bioimage-io.

    Parameters
    ----------
    model: StarDist2D, StarDist3D
        the model to convert
    outpath: str, Path
        where to save the model
    test_input: np.ndarray
        input image for generating test data
    test_input_axes: str or None
        the axes of the test input, for example 'YX' for a 2d image or 'ZYX' for a 3d volume
        using None assumes that axes of test_input are the same as those of model
    test_input_norm_axes: str
        the axes of the test input which will be jointly normalized, for example 'ZYX' for all spatial dimensions ('Z' ignored for 2D input)
        use 'ZYXC' to also jointly normalize channels (e.g. for RGB input images)
    name: str
        the name of this model (default: None)
        if None, uses the (folder) name of the model (i.e. `model.name`)
    mode: str
        the export type for this model (default: "tensorflow_saved_model_bundle")
    min_percentile: float
        min percentile to be used for image normalization (default: 1.0)
    max_percentile: float
        max percentile to be used for image normalization (default: 99.8)
    overwrite_spec_kwargs: dict or None
        spec keywords that should be overloaded (default: None)
    """
    _, build_model, *_ = _import()
    from .models import StarDist2D, StarDist3D
    isinstance(model, (StarDist2D, StarDist3D)) or _raise(ValueError("not a valid model"))
    0 <= min_percentile < max_percentile <= 100 or _raise(ValueError("invalid percentile values"))

    if name is None:
        name = model.name
    name = str(name)

    outpath = Path(outpath)
    if outpath.suffix == "":
        outdir = outpath
        zip_path = outdir / f"{name}.zip"
    elif outpath.suffix == ".zip":
        outdir = outpath.parent
        zip_path = outpath
    else:
        raise ValueError(f"outpath has to be a folder or zip file, got {outpath}")
    outdir.mkdir(exist_ok=True, parents=True)

    with tempfile.TemporaryDirectory() as _tmp_dir:
        tmp_dir = Path(_tmp_dir)
        kwargs = _get_stardist_metadata(tmp_dir, model)
        model_kwargs = _get_weights_and_model_metadata(tmp_dir, model, test_input, test_input_axes, test_input_norm_axes, mode,
                                                       min_percentile=min_percentile, max_percentile=max_percentile)
        kwargs.update(model_kwargs)
        if overwrite_spec_kwargs is not None:
            kwargs.update(overwrite_spec_kwargs)

        build_model(name=name, output_path=zip_path, add_deepimagej_config=(model.config.n_dim==2), root=tmp_dir, **kwargs)
        print(f"\nbioimage.io model with name '{name}' exported to '{zip_path}'")


def import_bioimageio(source, outpath):
    """Import stardist model from bioimage.io format, https://github.com/bioimage-io/spec-bioimage-io.

    Load a model in bioimage.io format from the given `source` (e.g. path to zip file, URL)
    and convert it to a regular stardist model, which will be saved in the folder `outpath`.

    Parameters
    ----------
    source: str, Path
        bioimage.io resource (e.g. path, URL)
    outpath: str, Path
        folder to save the stardist model (must not exist previously)

    Returns
    -------
    StarDist2D or StarDist3D
        stardist model loaded from `outpath`

    """
    import shutil, uuid
    from csbdeep.utils import save_json
    from .models import StarDist2D, StarDist3D
    *_, bioimageio_core, _ = _import()

    outpath = Path(outpath)
    not outpath.exists() or _raise(FileExistsError(f"'{outpath}' already exists"))

    with tempfile.TemporaryDirectory() as _tmp_dir:
        tmp_dir = Path(_tmp_dir)
        # download the full model content to a temporary folder
        zip_path = tmp_dir / f"{str(uuid.uuid4())}.zip"
        bioimageio_core.export_resource_package(source, output_path=zip_path)
        with ZipFile(zip_path, "r") as zip_ref:
            zip_ref.extractall(tmp_dir)
        zip_path.unlink()
        rdf_path = tmp_dir / "rdf.yaml"
        biomodel = bioimageio_core.load_resource_description(rdf_path)

        # read the stardist specific content
        'stardist' in biomodel.config or _raise(RuntimeError("bioimage.io model not compatible"))
        config = biomodel.config['stardist']['config']
        thresholds = biomodel.config['stardist']['thresholds']
        weights = biomodel.config['stardist']['weights']

        # make sure that the keras weights are in the attachments
        weights_file = None
        for f in biomodel.attachments.files:
            if f.name == weights and f.exists():
                weights_file = f
                break
        weights_file is not None or _raise(FileNotFoundError(f"couldn't find weights file '{weights}'"))

        # save the config and threshold to json, and weights to hdf5 to enable loading as stardist model
        # copy bioimageio files to separate sub-folder
        outpath.mkdir(parents=True)
        save_json(config, str(outpath / 'config.json'))
        save_json(thresholds, str(outpath / 'thresholds.json'))
        shutil.copy(str(weights_file), str(outpath / "weights_bioimageio.h5"))
        shutil.copytree(str(tmp_dir), str(outpath / "bioimageio"))

    model_class = (StarDist2D if config['n_dim'] == 2 else StarDist3D)
    model = model_class(None, outpath.name, basedir=str(outpath.parent))

    return model