Repository: NPLinker/nplinker (language: python)

Issue:
implement the function to generate `refseq id ↔ BGC id` mapping file
The function `generate_genome_bgc_mappings_file` should have following features:
- detect `extracted/antismash` folder and its subfolders and files
- extract `refseq id` from the name of subfolders detected
- extract `BGC id` from the name of .gbk files detected
- link the `refseq id` and `BGC id`
- output the links to a file named as `genome_bgc_mappings.json`  
clean up loader and downloader
This PR intends to do some cleanup during the process of checking how the strain mappings file (`strain_mappings.csv`) is generated, as mentioned in #148. The cleanup will make subsequent refactoring easier.

## Notice
1. Many commits are merged from PR #142, you can ignore them. **The first commit of this PR is https://github.com/NPLinker/nplinker/pull/149/commits/979e8c1a8d99d34b862ab04c8dc6acc8b604ae09.**
2. It's easier to review this PR commit by commit. The commit has meaningful title and/or explanation
Use UUID for singleton molecular family?
The issue is created from  https://github.com/NPLinker/nplinker/pull/142#discussion_r1203825955

> The -1 ID somewhat represents an unknown family or singleton family - all of these will have the same id, which is in conflict with the idea of having unique string id's for every object.
> Given the id in the GNPS output is -1, do we want to configure a UUID as ID to be able to distinguish the singletons?

- [x] check the influence of using UUID, e.g. how to verify the singletons
- [x] if feasible, using [UUID](https://docs.python.org/3/library/uuid.html) for singletons

- [x] does it make sense to use UUID for all types of MFs? not for now.


Discussion / hints:

Fix patch:
diff --git a/src/nplinker/annotations.py b/src/nplinker/annotations.py
index 6449b6d2..8ea288fd 100644
--- a/src/nplinker/annotations.py
+++ b/src/nplinker/annotations.py
@@ -33,6 +33,7 @@ def _headers_match_gnps(headers: list[str]) -> bool:
             return False
     return True
 
+
 @deprecated(version="1.3.3", reason="Use GNPSAnnotationLoader class instead.")
 def create_gnps_annotation(spec: Spectrum, gnps_anno: dict):
     """Function to insert png, json, svg and spectrum information in GNPS annotation data for given spectrum.
@@ -46,8 +47,8 @@ def create_gnps_annotation(spec: Spectrum, gnps_anno: dict):
     """
     # also insert useful URLs
     for t in ['png', 'json', 'svg', 'spectrum']:
-        gnps_anno[f'{t}_url'] = GNPS_URL_FORMAT.format(
-            t, gnps_anno['SpectrumID'])
+        gnps_anno[f'{t}_url'] = GNPS_URL_FORMAT.format(t,
+                                                       gnps_anno['SpectrumID'])
 
     if GNPS_KEY in spec.annotations:
         # TODO is this actually an error or can it happen normally?
@@ -60,7 +61,9 @@ def create_gnps_annotation(spec: Spectrum, gnps_anno: dict):
 
 
 @deprecated(version="1.3.3", reason="Use GNPSAnnotationLoader class instead.")
-def load_annotations(root: str | os.PathLike, config: str | os.PathLike, spectra: list[Spectrum], spec_dict: dict[str, Spectrum]) -> list[Spectrum]:
+def load_annotations(root: str | os.PathLike, config: str | os.PathLike,
+                     spectra: list[Spectrum],
+                     spec_dict: dict[str, Spectrum]) -> list[Spectrum]:
     """Load the annotations from the GNPS annotation file present in root to the spectra.
 
     Args:
@@ -120,8 +123,7 @@ def load_annotations(root: str | os.PathLike, config: str | os.PathLike, spectra
                     data = {}
                     for dc in data_cols:
                         if dc not in headers:
-                            logger.warning(
-                                f'Column lookup failed for "{dc}"')
+                            logger.warning(f'Column lookup failed for "{dc}"')
                             continue
                         data[dc] = line[headers.index(dc)]
 
@@ -160,8 +162,7 @@ def load_annotations(root: str | os.PathLike, config: str | os.PathLike, spectra
                     data = {}
                     for dc in data_cols:
                         if dc not in headers:
-                            logger.warning(
-                                f'Column lookup failed for "{dc}"')
+                            logger.warning(f'Column lookup failed for "{dc}"')
                             continue
                         data[dc] = line[headers.index(dc)]
                     spec_annotations[spec].append(data)
@@ -173,6 +174,7 @@ def load_annotations(root: str | os.PathLike, config: str | os.PathLike, spectra
 
     return spectra
 
+
 def _find_annotation_files(root: str, config: str) -> list[str]:
     """Detect all annotation files in the root folder or specified in the config file.
 
@@ -190,6 +192,7 @@ def _find_annotation_files(root: str, config: str) -> list[str]:
         annotation_files.append(os.path.join(root, f))
     return annotation_files
 
+
 def _read_config(config):
     ac = {}
     if os.path.exists(config):
diff --git a/src/nplinker/config.py b/src/nplinker/config.py
index 791dbe74..4b0c6fbc 100644
--- a/src/nplinker/config.py
+++ b/src/nplinker/config.py
@@ -1,20 +1,6 @@
-# Copyright 2021 The NPLinker Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import argparse
-import os
 from collections.abc import Mapping
+import os
 from shutil import copyfile
 import toml
 from xdg import XDG_CONFIG_HOME
@@ -143,24 +129,39 @@ def update(d, u):
             return d
 
         config = update(config, config_dict)
+        self._validate(config)
+        self.config = config
 
+    def _validate(self, config: dict) -> None:
+        """Validates the configuration dictionary to ensure that all required
+            fields are present and have valid values.
+
+        Args:
+            config (dict): The configuration dictionary to validate.
+
+        Raises:
+            ValueError: If the configuration dictionary is missing required
+                fields or contains invalid values.
+        """
         if 'dataset' not in config:
-            raise Exception('No dataset defined in configuration!')
+            raise ValueError('Not found config for "dataset".')
 
-        root = config['dataset']['root']
-        config['dataset']['platform_id'] = ''  # placeholder
+        root = config['dataset'].get('root')
+        if root is None:
+            raise ValueError('Not found config for "root".')
 
-        # check if the root has the special 'platform:' prefix and extract the
-        # ID if so. otherwise treat as a path
         if root.startswith('platform:'):
             config['dataset']['platform_id'] = root.replace('platform:', '')
-            logger.info('Selected platform project ID {}'.format(
-                config['dataset']['platform_id']))
+            logger.info('Loading from platform project ID %s',
+                        config['dataset']['platform_id'])
         else:
-            if root is None or not os.path.exists(root):
-                raise Exception(
-                    'Dataset path "{}" not found or not accessible'.format(
-                        root))
-            logger.info(f'Loading from local data in directory {root}')
-
-        self.config = config
+            config['dataset']['platform_id'] = ''
+            logger.info('Loading from local data in directory %s', root)
+
+        antismash = config['dataset'].get('antismash')
+        allowed_antismash_formats = ["default", "flat"]
+        if antismash is not None:
+            if 'format' in antismash and antismash[
+                    'format'] not in allowed_antismash_formats:
+                raise ValueError(
+                    f'Unknown antismash format: {antismash["format"]}')
diff --git a/src/nplinker/genomics/__init__.py b/src/nplinker/genomics/__init__.py
index 7f16a744..525c3015 100644
--- a/src/nplinker/genomics/__init__.py
+++ b/src/nplinker/genomics/__init__.py
@@ -1,15 +1,17 @@
 import logging
-
 from .abc import BGCLoaderBase
 from .bgc import BGC
 from .gcf import GCF
 from .genomics import filter_mibig_only_gcf
+from .genomics import generate_genome_bgc_mappings_file
+from .genomics import GENOME_BGC_MAPPINGS_FILENAME
 from .genomics import get_bgcs_from_gcfs
 from .genomics import get_strains_from_bgcs
 from .genomics import load_gcfs
 from .genomics import map_bgc_to_gcf
 from .genomics import map_strain_to_bgc
 
+
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
 __all__ = [
@@ -17,6 +19,8 @@
     "BGC",
     "GCF",
     "filter_mibig_only_gcf",
+    "generate_genome_bgc_mappings_file",
+    "GENOME_BGC_MAPPINGS_FILENAME",
     "get_bgcs_from_gcfs",
     "get_strains_from_bgcs",
     "load_gcfs",
diff --git a/src/nplinker/genomics/genomics.py b/src/nplinker/genomics/genomics.py
index 2ed1ab62..751dfd29 100644
--- a/src/nplinker/genomics/genomics.py
+++ b/src/nplinker/genomics/genomics.py
@@ -1,16 +1,62 @@
 from __future__ import annotations
 import csv
+import json
 from os import PathLike
 from pathlib import Path
 from deprecated import deprecated
 from nplinker.logconfig import LogConfig
 from nplinker.strain_collection import StrainCollection
+from nplinker.utils import list_dirs
+from nplinker.utils import list_files
 from .bgc import BGC
 from .gcf import GCF
 
 
 logger = LogConfig.getLogger(__name__)
 
+GENOME_BGC_MAPPINGS_FILENAME = "genome_bgc_mappings.json"
+
+
+def generate_genome_bgc_mappings_file(bgc_dir: str | PathLike) -> None:
+    """Generate a file that maps genome id to BGC id.
+
+    The output file is named in variable `GENOME_BGC_MAPPINGS_FILENAME` and
+    is placed in the same directory as `bgc_dir`. The file will be overwritten
+    if it already exists.
+
+    Args:
+        bgc_dir(str | PathLike): The directory has one-layer of subfolders and
+            each subfolder contains BGC files in `.gbk` format.
+            It assumes that
+            - the subfolder name is the genome id (e.g. refseq),
+            - the BGC file name is the BGC id.
+    """
+    bgc_dir = Path(bgc_dir)
+    genome_bgc_mappings = {}
+
+    for subdir in list_dirs(bgc_dir):
+        genome_id = Path(subdir).name
+        bgc_files = list_files(subdir, suffix=(".gbk"), keep_parent=False)
+        bgc_ids = [
+            bgc_id for f in bgc_files if (bgc_id := Path(f).stem) != genome_id
+        ]
+        genome_bgc_mappings[genome_id] = bgc_ids
+
+    # sort mappings by genome_id and construct json data
+    genome_bgc_mappings = dict(sorted(genome_bgc_mappings.items()))
+    json_data = [{
+        "genome_ID": k,
+        "BGC_ID": v
+    } for k, v in genome_bgc_mappings.items()]
+    json_data = {
+        "mappings": json_data,
+        "count": len(json_data),
+        "version": "1.0"
+    }
+
+    with open(bgc_dir / GENOME_BGC_MAPPINGS_FILENAME, "w") as f:
+        json.dump(json_data, f)
+
 
 def map_strain_to_bgc(strains: StrainCollection, bgcs: list[BGC],
                       bgc_genome_mapping: dict[str, str]):
@@ -36,7 +82,8 @@ def map_strain_to_bgc(strains: StrainCollection, bgcs: list[BGC],
             genome_id = bgc_genome_mapping[bgc.bgc_id]
         except KeyError as e:
             raise KeyError(
-                f"Not found BGC id {bgc.bgc_id} in BGC-genome mappings.") from e
+                f"Not found BGC id {bgc.bgc_id} in BGC-genome mappings."
+            ) from e
         try:
             strain = strains.lookup(genome_id)
         except KeyError as e:
diff --git a/src/nplinker/genomics/mibig/mibig_loader.py b/src/nplinker/genomics/mibig/mibig_loader.py
index 0be9f889..827fd709 100644
--- a/src/nplinker/genomics/mibig/mibig_loader.py
+++ b/src/nplinker/genomics/mibig/mibig_loader.py
@@ -31,7 +31,7 @@ def get_bgc_genome_mapping(self) -> dict[str, str]:
 
         Note that for MIBiG BGC, same value is used for BGC id and genome id.
         Users don't have to provide genome id for MIBiG BGCs in the
-        `strain_mapping.csv` file.
+        `strain_mappings.csv` file.
 
         Returns:
             dict[str, str]: key is BGC id/accession, value is
diff --git a/src/nplinker/loader.py b/src/nplinker/loader.py
index b25952c4..6de3bb2b 100644
--- a/src/nplinker/loader.py
+++ b/src/nplinker/loader.py
@@ -1,29 +1,15 @@
-# Copyright 2021 The NPLinker Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import glob
 import os
-from pathlib import Path
 import sys
+from pathlib import Path
 from nplinker.annotations import load_annotations
 from nplinker.class_info.chem_classes import ChemClassPredictions
 from nplinker.class_info.class_matches import ClassMatches
 from nplinker.class_info.runcanopus import run_canopus
 from nplinker.genomics import load_gcfs
 from nplinker.genomics.antismash import AntismashBGCLoader
-from nplinker.genomics.mibig import download_and_extract_mibig_metadata
 from nplinker.genomics.mibig import MibigBGCLoader
+from nplinker.genomics.mibig import download_and_extract_mibig_metadata
 from nplinker.logconfig import LogConfig
 from nplinker.metabolomics.metabolomics import load_dataset
 from nplinker.pairedomics.downloader import PODPDownloader
@@ -38,73 +24,11 @@
 
 logger = LogConfig.getLogger(__name__)
 
-
-def find_via_glob(path, file_type, optional=False):
-    try:
-        filename = glob.glob(path)[0]
-        return filename
-    except (OSError, IndexError) as e:
-        if not optional:
-            # "from None" suppresses the traceback for the original exception, which isn't really needed
-            raise Exception('ERROR: unable to find {} in path "{}"'.format(
-                file_type, path)) from None
-
-        logger.warn('WARNING: unable to find {} in path "{}"'.format(
-            file_type, path))
-        return None
-
-
-def find_via_glob_alts_dir(paths, file_type, optional=False):
-    path = None
-    for p in paths:
-        if os.path.exists(p):
-            path = p
-            break
-
-    if path is None and not optional:
-        raise Exception('ERROR: unable to find {} in {} paths: ({})'.format(
-            file_type, len(paths), paths))
-    elif path is None:
-        logger.warning('WARNING: unable to find {} in {} paths: ({})'.format(
-            file_type, len(paths), paths))
-
-    return path
-
-
-def find_via_glob_alts(paths, file_type, optional=False):
-    filename = None
-    for path in paths:
-        try:
-            filename = glob.glob(path)[0]
-            break
-        except (OSError, IndexError) as e:
-            continue
-
-    if filename is None and not optional:
-        raise Exception('ERROR: unable to find {} in {} paths: ({})'.format(
-            file_type, len(paths), paths))
-    elif filename is None:
-        logger.warning('WARNING: unable to find {} in {} paths: ({})'.format(
-            file_type, len(paths), paths))
-
-    return filename
-
-
-def find_bigscape_dir(broot):
-    logger.info(f'Trying to discover correct bigscape directory under {broot}')
-    for root, _, files in os.walk(broot):
-        if 'Network_Annotations_Full.tsv' in files:
-            logger.info(f'Found network files directory: {root}')
-            return root
-
-    return None
+NPLINKER_APP_DATA_DIR = files('nplinker').joinpath('data')
 
 
 class DatasetLoader():
 
-    ANTISMASH_FMT_DEFAULT = 'default'
-    ANTISMASH_FMT_FLAT = 'flat'
-    ANTISMASH_FMTS = [ANTISMASH_FMT_DEFAULT, ANTISMASH_FMT_FLAT]
     ANTISMASH_DELIMITERS_DEFAULT = ['.', '_', '-']
     ANTISMASH_IGNORE_SPACES_DEFAULT = False
 
@@ -155,49 +79,144 @@ class DatasetLoader():
     PFAM_PATH = os.path.join(sys.prefix, 'nplinker_lib')
 
     def __init__(self, config_data):
-        self._config = config_data
-        self._dataset = config_data['dataset']
-        self._docker = config_data.get('docker', {})
-        self._webapp = config_data.get('webapp', {})
-        self._antismash = config_data.get('antismash', {})
-        self._overrides = self._dataset.get('overrides', {})
-        self._antismash_delimiters = self._antismash.get(
+        # load the config data
+        self._config_dataset = config_data['dataset']
+        self._config_docker = config_data.get('docker', {})
+        self._config_webapp = config_data.get('webapp', {})
+        self._config_antismash = config_data.get('antismash', {})
+        self._config_overrides = self._config_dataset.get('overrides', {})
+        # set private attributes
+        self._antismash_delimiters = self._config_antismash.get(
             'antismash_delimiters', self.ANTISMASH_DELIMITERS_DEFAULT)
-        self._antismash_format = self._antismash.get(
-            'antismash_format', self.ANTISMASH_FMT_DEFAULT)
-        self._antismash_ignore_spaces = self._antismash.get(
+        self._antismash_ignore_spaces = self._config_antismash.get(
             'ignore_spaces', self.ANTISMASH_IGNORE_SPACES_DEFAULT)
-        self._bigscape_cutoff = self._dataset.get('bigscape_cutoff',
-                                                  self.BIGSCAPE_CUTOFF_DEFAULT)
-        self._extended_metadata_table_parsing = self._dataset.get(
+        self._bigscape_cutoff = self._config_dataset.get(
+            'bigscape_cutoff', self.BIGSCAPE_CUTOFF_DEFAULT)
+        self._extended_metadata_table_parsing = self._config_dataset.get(
             'extended_metadata_table_parsing',
             self.EXTENDED_METADATA_TABLE_PARSING_DEFAULT)
-        self._use_mibig = self._dataset.get('use_mibig',
-                                            self.USE_MIBIG_DEFAULT)
-        self._mibig_version = self._dataset.get('mibig_version',
-                                                self.MIBIG_VERSION_DEFAULT)
-
-        self._root = self._config['dataset']['root']
-        self._platform_id = self._config['dataset']['platform_id']
+        self._use_mibig = self._config_dataset.get('use_mibig',
+                                                   self.USE_MIBIG_DEFAULT)
+        self._mibig_version = self._config_dataset.get(
+            'mibig_version', self.MIBIG_VERSION_DEFAULT)
+        self._root = self._config_dataset['root']
+        self._platform_id = self._config_dataset['platform_id']
         self._remote_loading = len(self._platform_id) > 0
-        self.datadir = files('nplinker').joinpath('data')
+
+        # set public attributes
         self.dataset_id = os.path.split(
             self._root)[-1] if not self._remote_loading else self._platform_id
-
-        if self._remote_loading:
-            self._downloader = PODPDownloader(self._platform_id)
-        else:
-            self._downloader = None
-
         self.bgcs, self.gcfs, self.spectra, self.molfams = [], [], [], []
         self.mibig_bgc_dict = {}
         self.product_types = []
+        self.webapp_scoring_cutoff = self._config_webapp.get(
+            'tables_metcalf_threshold', self.TABLES_CUTOFF_DEFAULT)
+
         logger.debug('DatasetLoader({}, {}, {})'.format(
-            self._root, self._platform_id, self._remote_loading))
+            self._root, self.dataset_id, self._remote_loading))
+
+    def __repr__(self):
+        return 'Root={}\n   MGF={}\n   EDGES={}\n   NODES={}\n   BIGSCAPE={}\n   ANTISMASH={}\n'.format(
+            self._root, self.mgf_file, self.edges_file, self.nodes_file,
+            self.bigscape_dir, self.antismash_dir)
+
+    def validate(self):
+        """Download data and build paths for local data"""
+
+        # if remote loading mode, need to download the data here
+        # CG: for PODP workflow, strain_mappings.csv is generated in the download step
+        if self._remote_loading:
+            self._start_downloads()
+
+        # construct the paths and filenames required to load everything else and check
+        # they all seem to exist (but don't parse anything yet)
+        # TODO CG: the logics of _init_paths and _validate_paths are not clear
+        # 1. after downloading (manual preparation), some files alreay exist, some not
+        # 2. get the default, constructed or real path for each file/dir (need refactoring)
+        #   - self._config_overrides.get()
+        #   - os.path.join(self._root, 'strain_mappings.csv')
+        #   - find_via_glob() --> actually check if the file/dir exists
+        # 3. check if (some) file/dir exists
+        self._init_paths()
+        self._validate_paths()
+
+    def load(self):
+        # load strain mappings first
+        if not self._load_strain_mappings():
+            return False
+
+        if not self._load_metabolomics():
+            return False
+
+        if not self._load_genomics():
+            return False
+
+        if not self._load_class_info():
+            return False
+
+        self._load_optional()
+
+        # Restrict strain list to only relevant strains (those that are present
+        # in both genomic and metabolomic data)
+        # TODO add a config file option for this?
+        self._filter_only_common_strains()
+
+        # if the user specified a set of strains to be explicitly included, filter
+        # out everything except those strains
+        self._filter_user_strains()
+
+        # if we don't have at least *some* strains here it probably means missing mappings
+        # or a complete failure to parse things, so bail out
+        if len(self.strains) == 0:
+            raise Exception(
+                'Failed to find *ANY* strains, missing strain_mappings.csv?')
+
+        return True
+
+    def _start_downloads(self):
+        downloader = PODPDownloader(self._platform_id)
+        self._root = downloader.project_file_cache
+        logger.debug('remote loading mode, configuring root=%s', self._root)
+        # CG: to download both MET and GEN data
+        # CG: Continue to understand how strain_mappings.csv is generated
+        downloader.get(
+            self._config_docker.get('run_bigscape', self.RUN_BIGSCAPE_DEFAULT),
+            self._config_docker.get('extra_bigscape_parameters',
+                                    self.EXTRA_BIGSCAPE_PARAMS_DEFAULT),
+            self._use_mibig, self._mibig_version)
+
+    def _init_paths(self):
+        # 1. strain mapping are used for everything else so
+        self.strain_mappings_file = self._config_overrides.get(
+            self.OR_STRAINS) or os.path.join(self._root, 'strain_mappings.csv')
+
+        self._init_metabolomics_paths()
+
+        self._init_genomics_paths()
+
+        # 12. MISC: <root>/params.xml
+        self.params_file = os.path.join(self._root, 'params.xml')
+
+        # 13. MISC: <root>/description.txt
+        self.description_file = os.path.join(self._root, 'description.txt')
+
+        # 14. MISC: <root>/include_strains.csv / include_strains_file=<override>
+        self.include_strains_file = self._config_overrides.get(
+            self.OR_INCLUDE_STRAINS) or os.path.join(self._root,
+                                                     'include_strains.csv')
+
+        # 15. CLASS: <root>/canopus / canopus_dir=<override>
+        self.canopus_dir = self._config_overrides.get(
+            self.OR_CANOPUS) or os.path.join(self._root, 'canopus')
+
+        # 15. CLASS: <root>/canopus / canopus_dir=<override>
+        self.molnetenhancer_dir = self._config_overrides.get(
+            self.OR_MOLNETENHANCER) or os.path.join(self._root,
+                                                    'molnetenhancer')
 
     def _init_metabolomics_paths(self):
         # 2. MET: <root>/clusterinfo_summary/<some UID>.tsv (or .clustersummary apparently...) / nodes_file=<override>
-        self.nodes_file = self._overrides.get(
+        self.nodes_file = self._config_overrides.get(
             self.OR_NODES) or find_via_glob_alts([
                 os.path.join(self._root, 'file_mappings.tsv'),
                 os.path.join(self._root, 'clusterinfo*', '*.tsv'),
@@ -205,10 +224,11 @@ def _init_metabolomics_paths(self):
             ], self.OR_NODES)
 
         # 3. MET: <root>/networkedges_selfloop/<some UID>.selfloop (new) or .pairsinfo (old) / edges_file=<override>
-        self.edges_file = self._overrides.get(
+        self.edges_file = self._config_overrides.get(
             self.OR_EDGES) or find_via_glob_alts([
                 os.path.join(self._root, '*.pairsinfo'),
-                os.path.join(self._root, 'networkedges_selfloop', '*.pairsinfo'),
+                os.path.join(self._root, 'networkedges_selfloop',
+                             '*.pairsinfo'),
                 os.path.join(self._root, 'networkedges_selfloop', '*.selfloop')
             ], self.OR_EDGES)
 
@@ -216,28 +236,28 @@ def _init_metabolomics_paths(self):
         # TODO is the glob input OK?
         # => wait for updated dataset with latest output format
         # NOTE: only optional for Crusemann or Crusemann-like dataset format!
-        self.extra_nodes_file = self._overrides.get(
-            self.OR_EXTRA_NODES) or find_via_glob(
-                os.path.join(self._root, 'quantification_table_reformatted', '*.csv'),
+        self.extra_nodes_file = self._config_overrides.get(
+            self.OR_EXTRA_NODES) or find_via_glob(os.path.join(
+                self._root, 'quantification_table_reformatted', '*.csv'),
                                                   self.OR_EXTRA_NODES,
                                                   optional=True)
 
         # 5. MET: <root>/spectra/*.mgf (or <root>/*.mgf)/ mgf_file=<override>
-        self.mgf_file = self._overrides.get(self.OR_MGF) or find_via_glob_alts(
-            [
+        self.mgf_file = self._config_overrides.get(
+            self.OR_MGF) or find_via_glob_alts([
                 os.path.join(self._root, '*.mgf'),
                 os.path.join(self._root, 'spectra', '*.mgf')
             ], self.OR_MGF)
 
         # 6. MET: <root>/metadata_table/metadata_table-<number>.txt / metadata_table_file=<override>
-        self.metadata_table_file = self._overrides.get(
+        self.metadata_table_file = self._config_overrides.get(
             self.OR_METADATA) or find_via_glob(os.path.join(
                 self._root, 'metadata_table', 'metadata_table*.txt'),
                                                self.OR_METADATA,
                                                optional=True)
 
         # 7. MET: <root>/quantification_table/quantification_table-<number>.csv / quantification_table_file=<override>
-        self.quantification_table_file = self._overrides.get(
+        self.quantification_table_file = self._config_overrides.get(
             self.OR_QUANT) or find_via_glob(os.path.join(
                 self._root, 'quantification_table',
                 'quantification_table*.csv'),
@@ -247,30 +267,29 @@ def _init_metabolomics_paths(self):
         # 8. MET: <root>/DB_result/*.tsv (new) or <root>/result_specnets_DB/*.tsv (old) / annotations_dir=<override>
         if Path.is_file(Path(self._root) / "annotations.tsv"):
             self.annotations_dir = str(self._root)
-            self.annotations_config_file = os.path.join(self._root, "annotations.tsv")
+            self.annotations_config_file = os.path.join(
+                self._root, "annotations.tsv")
         else:
-            self.annotations_dir = self._overrides.get(
-                self.OR_ANNO) or find_via_glob_alts_dir(
-                    [
-                        os.path.join(self._root, 'DB_result'),
-                        os.path.join(self._root, 'result_specnets_DB'),
-                    ],
-                    self.OR_ANNO,
-                    optional=False
-                )
+            self.annotations_dir = self._config_overrides.get(
+                self.OR_ANNO) or find_via_glob_alts_dir([
+                    os.path.join(self._root, 'DB_result'),
+                    os.path.join(self._root, 'result_specnets_DB'),
+                ],
+                                                        self.OR_ANNO,
+                                                        optional=False)
             if self.annotations_dir is not None:
-                self.annotations_config_file = self._overrides.get(
-                    self.OR_ANNO_CONFIG) or os.path.join(self.annotations_dir,
-                                                         'annotations.tsv')
+                self.annotations_config_file = self._config_overrides.get(
+                    self.OR_ANNO_CONFIG) or os.path.join(
+                        self.annotations_dir, 'annotations.tsv')
 
     def _init_genomics_paths(self):
-         # 9. GEN: <root>/antismash / antismash_dir=<override>
-        self.antismash_dir = self._overrides.get(
+        # 9. GEN: <root>/antismash / antismash_dir=<override>
+        self.antismash_dir = self._config_overrides.get(
             self.OR_ANTISMASH) or os.path.join(self._root, 'antismash')
         self.antismash_cache = {}
 
         # 10. GEN: <root>/bigscape / bigscape_dir=<override>
-        self.bigscape_dir = self._overrides.get(
+        self.bigscape_dir = self._config_overrides.get(
             self.OR_BIGSCAPE) or os.path.join(self._root, 'bigscape')
         # what we really want here is the subdirectory containing the network/annotation files,
         # but in case this is the path to the top level bigscape output, try to figure it out automatically
@@ -283,300 +302,100 @@ def _init_genomics_paths(self):
                 self.bigscape_dir = new_bigscape_dir
 
         # 11. GEN: <root>/mibig_json / mibig_json_dir=<override>
-        self.mibig_json_dir = self._overrides.get(
+        self.mibig_json_dir = self._config_overrides.get(
             self.OR_MIBIG_JSON) or os.path.join(self._root, 'mibig_json')
 
-    def _init_paths(self):
-        # 1. strain mapping are used for everything else so
-        self.strain_mappings_file = self._overrides.get(
-            self.OR_STRAINS) or os.path.join(self._root, 'strain_mappings.csv')
-
-        self._init_metabolomics_paths()
-
-        self._init_genomics_paths()
-
-        # 12. MISC: <root>/params.xml
-        self.params_file = os.path.join(self._root, 'params.xml')
-
-        # 13. MISC: <root>/description.txt
-        self.description_file = os.path.join(self._root, 'description.txt')
-
-        # 14. MISC: <root>/include_strains.csv / include_strains_file=<override>
-        self.include_strains_file = self._overrides.get(
-            self.OR_INCLUDE_STRAINS) or os.path.join(self._root,
-                                                     'include_strains.csv')
-
-        # 15. CLASS: <root>/canopus / canopus_dir=<override>
-        self.canopus_dir = self._overrides.get(
-            self.OR_CANOPUS) or os.path.join(self._root, 'canopus')
-
-        # 15. CLASS: <root>/canopus / canopus_dir=<override>
-        self.molnetenhancer_dir = self._overrides.get(
-            self.OR_MOLNETENHANCER) or os.path.join(self._root,
-                                                    'molnetenhancer')
-    def _start_downloads(self):
-        self._root = self._downloader.project_file_cache
-        logger.debug('remote loading mode, configuring root={}'.format(
-                self._root))
-            # CG: to download both MET and GEN data
-        self._downloader.get(
-                self._docker.get('run_bigscape', self.RUN_BIGSCAPE_DEFAULT),
-                self._docker.get('extra_bigscape_parameters',
-                                 self.EXTRA_BIGSCAPE_PARAMS_DEFAULT),
-                self._use_mibig, self._mibig_version)
-
     def _validate_paths(self):
-        for f in self.required_paths():
-            if not os.path.exists(f):
-                raise FileNotFoundError(
-                    'File/directory "{}" does not exist or is not readable!'.
-                    format(f))
+        """Validates that the required files and directories exist before loading starts.
+        """
+        required_paths = [
+            self.nodes_file, self.edges_file, self.mgf_file, self.antismash_dir
+        ]
+        optional_paths = [self.annotations_dir]
 
-        for f in self.optional_paths():
-            if not os.path.exists(f):
-                logger.warning(
-                    'Optional file/directory "{}" does not exist or is not readable!'
-                    .format(f))
+        for f in required_paths:
+            if not os.path.exists(str(f)):
+                raise FileNotFoundError(
+                    f'File/directory "{f}" does not exist.')
 
-    def validate(self):
-        """Download data and build paths for local data"""
+        for f in optional_paths:
+            if not os.path.exists(str(f)):
+                logger.warning('Optional file/directory "%s" does not exist',
+                               f)
 
-        # check antismash format is recognised
-        if self._antismash_format not in self.ANTISMASH_FMTS:
-            raise Exception('Unknown antismash format: {}'.format(
-                self._antismash_format))
+    # TODO: this function should be refactored to Loader class
+    def _load_strain_mappings(self):
+        # this file should be a csv file, one line per strain, containing a list
+        # of possible alternative IDs (the first one being the preferred ID).
+        #
+        # this is a per-dataset mapping, and is then merged with the global mapping file
+        # packaged with nplinker itself
+        self._init_global_strain_id_mapping()
 
-        # if remote loading mode, need to download the data here
-        # CG: for PODP workflow, strain_mappings.csv is generated in the download step
-        if self._remote_loading:
-            self._start_downloads()
+        # now load the dataset mapping in the same way
+        # TODO: what happens in case of clashes (differing primary IDs?)
+        # CG: the `if` never happens for PODP pipeline; for non-PODP pipeline,
+        # self.strains is empty and will cause error.
+        # TODO: remove the `if` condition
+        if not os.path.exists(self.strain_mappings_file):
+            # create an empty placeholder file and show a warning
+            logger.warn(
+                'No strain_mappings.csv file found! Attempting to create one')
+            self.strains.generate_strain_mappings(self.strain_mappings_file,
+                                                  self.antismash_dir)
+            # raise Exception('Unable to load strain_mappings file: {}'.format(self.strain_mappings_file))
+        else:
+            self.strains.add_from_file(self.strain_mappings_file)
+            logger.info('Loaded dataset strain IDs ({} total)'.format(
+                len(self.strains)))
 
-        # construct the paths and filenames required to load everything else and check
-        # they all seem to exist (but don't parse anything yet)
-        self._init_paths()
-        self._validate_paths()
+        return True
 
-    def load(self, met_only):
-        # load strain mappings first
-        if not self._load_strain_mappings():
-            return False
+    def _init_global_strain_id_mapping(self):
+        """The global strain mapping is predefined by the NPLinker package.
 
-        if not self._load_metabolomics():
-            return False
+            See `src/nplinker/strain_id_mapping.csv`
+        """
+        self.strains = StrainCollection()
+        global_strain_id_file = NPLINKER_APP_DATA_DIR.joinpath(
+            'strain_id_mapping.csv')
+        self.strains.add_from_file(global_strain_id_file)
+        logger.info('Loaded global strain IDs ({} total)'.format(
+            len(self.strains)))
 
-        if not met_only and not self._load_genomics():
-            return False
+    # TODO CG: replace deprecated load_dataset with GPNSLoader
+    def _load_metabolomics(self):
+        spec_dict, self.spectra, self.molfams, unknown_strains = load_dataset(
+            self.strains, self.mgf_file, self.edges_file, self.nodes_file,
+            self.quantification_table_file, self.metadata_table_file,
+            self._extended_metadata_table_parsing)
 
-        if not self._load_class_info():
-            return False
+        us_path = os.path.join(self._root, 'unknown_strains_met.csv')
+        logger.warning(
+            'Writing unknown strains from METABOLOMICS data to {}'.format(
+                us_path))
+        with open(us_path, 'w') as us:
+            us.write('# unknown strain label\n')
+            for strain in unknown_strains.keys():
+                us.write(f'{strain}\n')
 
-        self._load_optional()
+        # load any available annotations from GNPS or user-provided files
+        logger.info('Loading provided annotation files ({})'.format(
+            self.annotations_dir))
+        self.spectra = load_annotations(self.annotations_dir,
+                                        self.annotations_config_file,
+                                        self.spectra, spec_dict)
+        return True
 
-        # Restrict strain list to only relevant strains (those that are present
-        # in both genomic and
-        if not met_only:
-            # TODO add a config file option for this?
-            self._filter_only_common_strains()
+    # CG: load gemonics data into memory
+    def _load_genomics(self):
+        # TODO split this method up a bit
 
-            # if the user specified a set of strains to be explicitly included, filter
-            # out everything except those strains
-            self._filter_user_strains()
-
-        # if we don't have at least *some* strains here it probably means missing mappings
-        # or a complete failure to parse things, so bail out
-        if len(self.strains) == 0:
-            raise Exception(
-                'Failed to find *ANY* strains, missing strain_mappings.csv?')
-
-        return True
-
-    def _filter_user_strains(self):
-        """
-        If the user has supplied a list of strains to be explicitly included, go through the
-        existing sets of objects we have and remove any that only include other strains. This
-        involves an initial round of removing BGC and Spectrum objects, then a further round
-        of removing now-empty GCF and MolFam objects.
-        """
-        if len(self.include_only_strains) == 0:
-            logger.info('No further strain filtering to apply')
-            return
-
-        logger.info(
-            'Found a list of {} strains to retain, filtering objects'.format(
-                len(self.include_only_strains)))
-
-        # filter the main list of strains
-        self.strains.filter(self.include_only_strains)
-
-        if len(self.strains) == 0:
-            logger.error(
-                'Strain list has been filtered down until it is empty! ')
-            logger.error(
-                'This probably indicates that you tried to specifically include a set of strains that had no overlap with the set common to metabolomics and genomics data (see the common_strains.csv in the dataset folder for a list of these'
-            )
-            raise Exception(
-                'No strains left after filtering, cannot continue!')
-
-        # get the list of BGCs which have a strain found in the set we were given
-        bgcs_to_retain = {
-            bgc
-            for bgc in self.bgcs if bgc.strain in self.include_only_strains
-        }
-        # get the list of spectra which have at least one strain in the set
-        spectra_to_retain = {
-            spec
-            for spec in self.spectra for sstrain in spec.strains
-            if sstrain in self.include_only_strains
-        }
-
-        logger.info('Current / filtered BGC counts: {} / {}'.format(
-            len(self.bgcs), len(bgcs_to_retain)))
-        logger.info('Current / filtered spectra counts: {} / {}'.format(
-            len(self.spectra), len(spectra_to_retain)))
-
-        self.bgcs = list(bgcs_to_retain)
-
-        self.spectra = list(spectra_to_retain)
-        # also need to filter the set of strains attached to each spectrum
-        for i, spec in enumerate(self.spectra):
-            spec.strains.filter(self.include_only_strains)
-            spec.id = i
-
-        # now filter GCFs and MolFams based on the filtered BGCs and Spectra
-        gcfs = {parent for bgc in self.bgcs for parent in bgc.parents}
-        logger.info('Current / filtered GCF counts: {} / {}'.format(
-            len(self.gcfs), len(gcfs)))
-        self.gcfs = list(gcfs)
-        # filter each GCF's strain list
-        for gcf in self.gcfs:
-            gcf.strains.filter(self.include_only_strains)
-
-        molfams = {spec.family for spec in self.spectra}
-        logger.info('Current / filtered MolFam counts: {} / {}'.format(
-            len(self.molfams), len(molfams)))
-        self.molfams = list(molfams)
-        for i, molfam in enumerate(self.molfams):
-            molfam.id = i
-
-    def _filter_only_common_strains(self):
-        """
-        Filter strain population to only strains present in both genomic and molecular data
-        """
-        # TODO: Maybe there should be an option to specify which strains are used, both so we can
-        #    selectively exclude strains, and include strains that are missing from either side.
-        bgc_strains = {x.strain for x in self.bgcs}
-        spectrum_strains = set().union(*[x.strains for x in self.spectra])
-        common_strains = bgc_strains.intersection(spectrum_strains)
-        logger.debug(
-            'Filtering strains: genomics count {}, metabolomics count: {}'.
-            format(len(bgc_strains), len(spectrum_strains)))
-        logger.debug(f'Common strains found: {len(common_strains)}')
-
-        # write out a list of the common strains to the dataset folder (might be useful for
-        # anyone wanting to do additional filtering)
-        cs_path = os.path.join(self._root, 'common_strains.csv')
-        logger.info(f'Writing common strain labels to {cs_path}')
-        with open(cs_path, 'w') as cs:
-            cs.write('# strain label\n')
-            for strain in self.strains:
-                cs.write(f'{strain.id}\n')
-
-        # filter the master list of strains down to include only the common set
-        self.strains.filter(common_strains)
-
-        for gcf in self.gcfs:
-            gcf.strains.filter(common_strains)
-        for spec in self.spectra:
-            spec.strains.filter(common_strains)
-        logger.info('Strains filtered down to total of {}'.format(
-            len(self.strains)))
-
-    def _load_optional(self):
-        self.gnps_params = {}
-        if os.path.exists(self.params_file):
-            logger.debug('Loading params.xml')
-            tree = ET.parse(self.params_file)
-            root = tree.getroot()
-            # this file has a simple structure:
-            # <parameters>
-            #   <parameter name="something">value</parameter>
-            # </parameters>
-            for param in root:
-                self.gnps_params[param.attrib['name']] = param.text
-
-            logger.debug(f'Parsed {len(self.gnps_params)} GNPS params')
-
-        self.description_text = '<no description>'
-        if os.path.exists(self.description_file):
-            self.description_text = open(self.description_file).read()
-            logger.debug('Parsed description text')
-
-        self.include_only_strains = set()
-        if os.path.exists(self.include_strains_file):
-            logger.debug('Loading include_strains from {}'.format(
-                self.include_strains_file))
-            strain_list = open(self.include_strains_file).readlines()
-            self.include_only_strains = StrainCollection()
-            for line_num, sid in enumerate(strain_list):
-                sid = sid.strip()  # get rid of newline
-                try:
-                    strain_obj = self.strains.lookup(sid)
-                except KeyError:
-                    logger.warning(
-                        'Line {} of {}: invalid/unknown strain ID "{}"'.format(
-                            line_num + 1, self.include_strains_file, sid))
-                    continue
-                self.include_only_strains.add(strain_obj)
-            logger.debug('Found {} strain IDs in include_strains'.format(
-                len(self.include_only_strains)))
-
-    def _load_genomics_extra(self):
-        if not os.path.exists(self.mibig_json_dir):
-            if self._use_mibig:
-                logger.info(
-                    'Attempting to download MiBIG JSON database (v{})...'.
-                    format(self._mibig_version))
-                download_and_extract_mibig_metadata(self._root,
-                                                    self.mibig_json_dir,
-                                                    self._mibig_version)
-            else:
-                logger.warning(
-                    'Not downloading MiBIG database automatically, use_mibig = false'
-                )
-
-        if not os.path.exists(self.bigscape_dir):
-            should_run_bigscape = self._docker.get('run_bigscape',
-                                                   self.RUN_BIGSCAPE_DEFAULT)
-            extra_bigscape_parameters = self._docker.get(
-                'extra_bigscape_parameters',
-                self.EXTRA_BIGSCAPE_PARAMS_DEFAULT)
-            if should_run_bigscape:
-                # TODO this should not be attempted if not in Docker env
-                logger.info(
-                    'Running BiG-SCAPE! extra_bigscape_parameters="{}"'.format(
-                        extra_bigscape_parameters))
-                try:
-                    run_bigscape('bigscape.py',
-                                 os.path.join(self._root, 'antismash'),
-                                 os.path.join(self._root, 'bigscape'),
-                                 self.PFAM_PATH,
-                                 extra_params=extra_bigscape_parameters)
-                except Exception as e:
-                    logger.warning(
-                        'Failed to run BiG-SCAPE on antismash data, error was "{}"'
-                        .format(e))
-
-                self.bigscape_dir = find_bigscape_dir(self.bigscape_dir)
-
-    # CG: load gemonics data into memory
-    def _load_genomics(self):
-        # TODO split this method up a bit
-
-        # hmmscan apparently can't cope with filenames that have spaces in them, and so if you try
-        # to run bigscape on a dataset like that it will just refuse. so, to try and avoid anyone
-        # having to manually fix this stupid problem, we will attempt to rename every .gbk file here
-        # TODO is this something nplinker should do, or dataset authors??
-        logger.debug("\nLoading genomics starts...")
+        # hmmscan apparently can't cope with filenames that have spaces in them, and so if you try
+        # to run bigscape on a dataset like that it will just refuse. so, to try and avoid anyone
+        # having to manually fix this stupid problem, we will attempt to rename every .gbk file here
+        # TODO is this something nplinker should do, or dataset authors??
+        logger.debug("\nLoading genomics starts...")
 
         #----------------------------------------------------------------------
         # CG: replace space with underscore for antismash folder and file names
@@ -652,11 +471,8 @@ def _load_genomics(self):
         #   and `get_strains_from_bgcs`
 
         self.gcfs, self.bgcs, self.strains, unknown_strains = load_gcfs(
-            self.bigscape_dir,
-            self.strains,
-            self.mibig_bgc_dict,
-            antismash_bgc_loader.get_bgcs(),
-            antismash_bgc_loader.get_files(),
+            self.bigscape_dir, self.strains, self.mibig_bgc_dict,
+            antismash_bgc_loader.get_bgcs(), antismash_bgc_loader.get_files(),
             self._bigscape_cutoff)
 
         #----------------------------------------------------------------------
@@ -674,34 +490,43 @@ def _load_genomics(self):
 
         return True
 
-    # TODO CG: replace deprecated load_dataset with GPNSLoader
-    def _load_metabolomics(self):
-        spec_dict, self.spectra, self.molfams, unknown_strains = load_dataset(
-            self.strains,
-            self.mgf_file,
-            self.edges_file,
-            self.nodes_file,
-            self.quantification_table_file,
-            self.metadata_table_file,
-            self._extended_metadata_table_parsing
-        )
+    def _load_genomics_extra(self):
+        if not os.path.exists(self.mibig_json_dir):
+            if self._use_mibig:
+                logger.info(
+                    'Attempting to download MiBIG JSON database (v{})...'.
+                    format(self._mibig_version))
+                download_and_extract_mibig_metadata(self._root,
+                                                    self.mibig_json_dir,
+                                                    self._mibig_version)
+            else:
+                logger.warning(
+                    'Not downloading MiBIG database automatically, use_mibig = false'
+                )
 
-        us_path = os.path.join(self._root, 'unknown_strains_met.csv')
-        logger.warning(
-            'Writing unknown strains from METABOLOMICS data to {}'.format(
-                us_path))
-        with open(us_path, 'w') as us:
-            us.write('# unknown strain label\n')
-            for strain in unknown_strains.keys():
-                us.write(f'{strain}\n')
+        if not os.path.exists(self.bigscape_dir):
+            should_run_bigscape = self._config_docker.get(
+                'run_bigscape', self.RUN_BIGSCAPE_DEFAULT)
+            extra_bigscape_parameters = self._config_docker.get(
+                'extra_bigscape_parameters',
+                self.EXTRA_BIGSCAPE_PARAMS_DEFAULT)
+            if should_run_bigscape:
+                # TODO this should not be attempted if not in Docker env
+                logger.info(
+                    'Running BiG-SCAPE! extra_bigscape_parameters="{}"'.format(
+                        extra_bigscape_parameters))
+                try:
+                    run_bigscape('bigscape.py',
+                                 os.path.join(self._root, 'antismash'),
+                                 os.path.join(self._root, 'bigscape'),
+                                 self.PFAM_PATH,
+                                 extra_params=extra_bigscape_parameters)
+                except Exception as e:
+                    logger.warning(
+                        'Failed to run BiG-SCAPE on antismash data, error was "{}"'
+                        .format(e))
 
-        # load any available annotations from GNPS or user-provided files
-        logger.info('Loading provided annotation files ({})'.format(
-            self.annotations_dir))
-        self.spectra = load_annotations(self.annotations_dir,
-                                        self.annotations_config_file,
-                                        self.spectra, spec_dict)
-        return True
+                self.bigscape_dir = find_bigscape_dir(self.bigscape_dir)
 
     def _load_class_info(self):
         """Load class match info (based on mibig) and chemical class predictions
@@ -713,14 +538,14 @@ def _load_class_info(self):
             True if everything completes
         """
         # load Class_matches with mibig info from data
-        mibig_class_file = self.datadir.joinpath(
+        mibig_class_file = NPLINKER_APP_DATA_DIR.joinpath(
             'MIBiG2.0_compounds_with_AS_BGC_CF_NPC_classes.txt')
         self.class_matches = ClassMatches(mibig_class_file)
 
         # run canopus if canopus_dir does not exist
-        should_run_canopus = self._docker.get('run_canopus',
-                                              self.RUN_CANOPUS_DEFAULT)
-        extra_canopus_parameters = self._docker.get(
+        should_run_canopus = self._config_docker.get('run_canopus',
+                                                     self.RUN_CANOPUS_DEFAULT)
+        extra_canopus_parameters = self._config_docker.get(
             'extra_canopus_parameters', self.EXTRA_CANOPUS_PARAMS_DEFAULT)
         if should_run_canopus:
             # don't run canopus when canopus dir exists already
@@ -760,59 +585,203 @@ def _load_class_info(self):
         self.chem_classes = chem_classes
         return True
 
-    # TODO: this function should be refactored to Loader class
-    def _load_strain_mappings(self):
-        # this file should be a csv file, one line per strain, containing a list
-        # of possible alternative IDs (the first one being the preferred ID).
-        #
-        # this is a per-dataset mapping, and is then merged with the global mapping file
-        # packaged with nplinker itself
-        self._init_global_strain_id_mapping()
+    def _load_optional(self):
+        self.gnps_params = {}
+        if os.path.exists(self.params_file):
+            logger.debug('Loading params.xml')
+            tree = ET.parse(self.params_file)
+            root = tree.getroot()
+            # this file has a simple structure:
+            # <parameters>
+            #   <parameter name="something">value</parameter>
+            # </parameters>
+            for param in root:
+                self.gnps_params[param.attrib['name']] = param.text
 
-        # now load the dataset mapping in the same way
-        # TODO: what happens in case of clashes (differing primary IDs?)
-        # CG: the `if` never happens for PODP pipeline; for non-PODP pipeline,
-        # self.strains is empty and will cause error.
-        # TODO: remove the `if` condition
-        if not os.path.exists(self.strain_mappings_file):
-            # create an empty placeholder file and show a warning
-            logger.warn(
-                'No strain_mappings.csv file found! Attempting to create one')
-            self.strains.generate_strain_mappings(self.strain_mappings_file,
-                                                  self.antismash_dir)
-            # raise Exception('Unable to load strain_mappings file: {}'.format(self.strain_mappings_file))
-        else:
-            self.strains.add_from_file(self.strain_mappings_file)
-            logger.info('Loaded dataset strain IDs ({} total)'.format(
-                len(self.strains)))
+            logger.debug(f'Parsed {len(self.gnps_params)} GNPS params')
 
-        return True
+        self.description_text = '<no description>'
+        if os.path.exists(self.description_file):
+            self.description_text = open(self.description_file).read()
+            logger.debug('Parsed description text')
 
-    def _init_global_strain_id_mapping(self):
-        """The global strain mapping is predefined by the NPLinker package.
+        self.include_only_strains = set()
+        if os.path.exists(self.include_strains_file):
+            logger.debug('Loading include_strains from {}'.format(
+                self.include_strains_file))
+            strain_list = open(self.include_strains_file).readlines()
+            self.include_only_strains = StrainCollection()
+            for line_num, sid in enumerate(strain_list):
+                sid = sid.strip()  # get rid of newline
+                try:
+                    strain_obj = self.strains.lookup(sid)
+                except KeyError:
+                    logger.warning(
+                        'Line {} of {}: invalid/unknown strain ID "{}"'.format(
+                            line_num + 1, self.include_strains_file, sid))
+                    continue
+                self.include_only_strains.add(strain_obj)
+            logger.debug('Found {} strain IDs in include_strains'.format(
+                len(self.include_only_strains)))
 
-            See `src/nplinker/strain_id_mapping.csv`
+    def _filter_only_common_strains(self):
         """
-        self.strains = StrainCollection()
-        global_strain_id_file = self.datadir.joinpath('strain_id_mapping.csv')
-        self.strains.add_from_file(global_strain_id_file)
-        logger.info('Loaded global strain IDs ({} total)'.format(
+        Filter strain population to only strains present in both genomic and molecular data
+        """
+        # TODO: Maybe there should be an option to specify which strains are used, both so we can
+        #    selectively exclude strains, and include strains that are missing from either side.
+        bgc_strains = {x.strain for x in self.bgcs}
+        spectrum_strains = set().union(*[x.strains for x in self.spectra])
+        common_strains = bgc_strains.intersection(spectrum_strains)
+        logger.debug(
+            'Filtering strains: genomics count {}, metabolomics count: {}'.
+            format(len(bgc_strains), len(spectrum_strains)))
+        logger.debug(f'Common strains found: {len(common_strains)}')
+
+        # write out a list of the common strains to the dataset folder (might be useful for
+        # anyone wanting to do additional filtering)
+        cs_path = os.path.join(self._root, 'common_strains.csv')
+        logger.info(f'Writing common strain labels to {cs_path}')
+        with open(cs_path, 'w') as cs:
+            cs.write('# strain label\n')
+            for strain in self.strains:
+                cs.write(f'{strain.id}\n')
+
+        # filter the master list of strains down to include only the common set
+        self.strains.filter(common_strains)
+
+        for gcf in self.gcfs:
+            gcf.strains.filter(common_strains)
+        for spec in self.spectra:
+            spec.strains.filter(common_strains)
+        logger.info('Strains filtered down to total of {}'.format(
             len(self.strains)))
 
-    def required_paths(self):
-        # these are files/paths that *must* exist for loading to begin
-        return [
-            self.nodes_file, self.edges_file, self.mgf_file, self.antismash_dir
-        ]
+    def _filter_user_strains(self):
+        """
+        If the user has supplied a list of strains to be explicitly included, go through the
+        existing sets of objects we have and remove any that only include other strains. This
+        involves an initial round of removing BGC and Spectrum objects, then a further round
+        of removing now-empty GCF and MolFam objects.
+        """
+        if len(self.include_only_strains) == 0:
+            logger.info('No further strain filtering to apply')
+            return
+
+        logger.info(
+            'Found a list of {} strains to retain, filtering objects'.format(
+                len(self.include_only_strains)))
 
-    def optional_paths(self):
-        return [self.annotations_dir]
+        # filter the main list of strains
+        self.strains.filter(self.include_only_strains)
 
-    def webapp_scoring_cutoff(self):
-        return self._webapp.get('tables_metcalf_threshold',
-                                self.TABLES_CUTOFF_DEFAULT)
+        if len(self.strains) == 0:
+            logger.error(
+                'Strain list has been filtered down until it is empty! ')
+            logger.error(
+                'This probably indicates that you tried to specifically include a set of strains that had no overlap with the set common to metabolomics and genomics data (see the common_strains.csv in the dataset folder for a list of these'
+            )
+            raise Exception(
+                'No strains left after filtering, cannot continue!')
 
-    def __repr__(self):
-        return 'Root={}\n   MGF={}\n   EDGES={}\n   NODES={}\n   BIGSCAPE={}\n   ANTISMASH={}\n'.format(
-            self._root, self.mgf_file, self.edges_file, self.nodes_file,
-            self.bigscape_dir, self.antismash_dir)
+        # get the list of BGCs which have a strain found in the set we were given
+        bgcs_to_retain = {
+            bgc
+            for bgc in self.bgcs if bgc.strain in self.include_only_strains
+        }
+        # get the list of spectra which have at least one strain in the set
+        spectra_to_retain = {
+            spec
+            for spec in self.spectra for sstrain in spec.strains
+            if sstrain in self.include_only_strains
+        }
+
+        logger.info('Current / filtered BGC counts: {} / {}'.format(
+            len(self.bgcs), len(bgcs_to_retain)))
+        logger.info('Current / filtered spectra counts: {} / {}'.format(
+            len(self.spectra), len(spectra_to_retain)))
+
+        self.bgcs = list(bgcs_to_retain)
+
+        self.spectra = list(spectra_to_retain)
+        # also need to filter the set of strains attached to each spectrum
+        for i, spec in enumerate(self.spectra):
+            spec.strains.filter(self.include_only_strains)
+            spec.id = i
+
+        # now filter GCFs and MolFams based on the filtered BGCs and Spectra
+        gcfs = {parent for bgc in self.bgcs for parent in bgc.parents}
+        logger.info('Current / filtered GCF counts: {} / {}'.format(
+            len(self.gcfs), len(gcfs)))
+        self.gcfs = list(gcfs)
+        # filter each GCF's strain list
+        for gcf in self.gcfs:
+            gcf.strains.filter(self.include_only_strains)
+
+        molfams = {spec.family for spec in self.spectra}
+        logger.info('Current / filtered MolFam counts: {} / {}'.format(
+            len(self.molfams), len(molfams)))
+        self.molfams = list(molfams)
+        for i, molfam in enumerate(self.molfams):
+            molfam.id = i
+
+
+def find_via_glob(path, file_type, optional=False):
+    try:
+        filename = glob.glob(path)[0]
+        return filename
+    except (OSError, IndexError) as e:
+        if not optional:
+            # "from None" suppresses the traceback for the original exception, which isn't really needed
+            raise Exception('ERROR: unable to find {} in path "{}"'.format(
+                file_type, path)) from None
+
+        logger.warn('WARNING: unable to find {} in path "{}"'.format(
+            file_type, path))
+        return None
+
+
+def find_via_glob_alts_dir(paths, file_type, optional=False):
+    path = None
+    for p in paths:
+        if os.path.exists(p):
+            path = p
+            break
+
+    if path is None and not optional:
+        raise Exception('ERROR: unable to find {} in {} paths: ({})'.format(
+            file_type, len(paths), paths))
+    elif path is None:
+        logger.warning('WARNING: unable to find {} in {} paths: ({})'.format(
+            file_type, len(paths), paths))
+
+    return path
+
+
+def find_via_glob_alts(paths, file_type, optional=False):
+    filename = None
+    for path in paths:
+        try:
+            filename = glob.glob(path)[0]
+            break
+        except (OSError, IndexError) as e:
+            continue
+
+    if filename is None and not optional:
+        raise Exception('ERROR: unable to find {} in {} paths: ({})'.format(
+            file_type, len(paths), paths))
+    elif filename is None:
+        logger.warning('WARNING: unable to find {} in {} paths: ({})'.format(
+            file_type, len(paths), paths))
+
+    return filename
+
+
+def find_bigscape_dir(broot):
+    logger.info(f'Trying to discover correct bigscape directory under {broot}')
+    for root, _, files in os.walk(broot):
+        if 'Network_Annotations_Full.tsv' in files:
+            logger.info(f'Found network files directory: {root}')
+            return root
+
+    return None
diff --git a/src/nplinker/nplinker.py b/src/nplinker/nplinker.py
index 6e58981f..747dc989 100644
--- a/src/nplinker/nplinker.py
+++ b/src/nplinker/nplinker.py
@@ -3,10 +3,10 @@
 import logging
 import sys
 from typing import TYPE_CHECKING
-from .config import Args
 from .config import Config
 from .genomics import BGC
 from .genomics import GCF
+from .loader import NPLINKER_APP_DATA_DIR
 from .loader import DatasetLoader
 from .logconfig import LogConfig
 from .metabolomics.molecular_family import MolecularFamily
@@ -216,7 +216,7 @@ def dataset_id(self):
     @property
     def data_dir(self):
         """Returns path to nplinker/data directory (files packaged with the app itself)"""
-        return self._loader.datadir
+        return NPLINKER_APP_DATA_DIR
 
     @property
     def gnps_params(self):
@@ -244,7 +244,7 @@ def bigscape_cutoff(self):
         """Returns the current BiGSCAPE clustering cutoff value"""
         return self._loader._bigscape_cutoff
 
-    def load_data(self, new_bigscape_cutoff=None, met_only=False):
+    def load_data(self, new_bigscape_cutoff=None):
         """Loads the basic components of a dataset.
 
         This method is responsible for loading the various pieces of the supplied dataset into
@@ -255,20 +255,14 @@ def load_data(self, new_bigscape_cutoff=None, met_only=False):
         Returns:
             bool: True if successful, False otherwise
         """
-        # TODO: the met_only is useless, remove it. NPlinker will stop working if met_only=True
-        # typical case where load_data is being called with no params
+        logger.debug('load_data(new_bigscape_cutoff=%s)', new_bigscape_cutoff)
         if new_bigscape_cutoff is None:
-            logger.debug(
-                'load_data (normal case, full load, met_only={})'.format(
-                    met_only))
             self._loader.validate()
-
-            if not self._loader.load(met_only=met_only):
+            if not self._loader.load():
                 return False
         else:
             # CG: only reload genomics data when changing bigscape cutoff
             # TODO: this part should be removed, reload everything if bigscape data changes.
-            logger.debug(f'load_data with new cutoff = {new_bigscape_cutoff}')
             # 1. change the cutoff (which by itself doesn't do anything)
             self._loader._bigscape_cutoff = new_bigscape_cutoff
             # 2. reload the strain mappings (MiBIG filtering may have removed strains
diff --git a/src/nplinker/pairedomics/downloader.py b/src/nplinker/pairedomics/downloader.py
index 1ee6942a..b5fb3362 100644
--- a/src/nplinker/pairedomics/downloader.py
+++ b/src/nplinker/pairedomics/downloader.py
@@ -1,24 +1,10 @@
-# Copyright 2021 The NPLinker Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import json
 import os
 import shutil
 import sys
 import zipfile
-import httpx
 from deprecated import deprecated
+import httpx
 from progress.spinner import Spinner
 from nplinker.genomics.mibig import download_and_extract_mibig_metadata
 from nplinker.logconfig import LogConfig
@@ -37,8 +23,6 @@
 
 MIBIG_METADATA_URL = 'https://dl.secondarymetabolites.org/mibig/mibig_json_{}.tar.gz'
 MIBIG_BGC_METADATA_URL = 'https://mibig.secondarymetabolites.org/repository/{}/annotations.json'
-# MIBIG_BGC_GENBANK_URL = 'https://mibig.secondarymetabolites.org/repository/{}/{}.gbk'
-# MIBIG_BGC_JSON_URL = 'https://mibig.secondarymetabolites.org/repository/{}/{}.json'
 
 
 class PODPDownloader():
@@ -51,7 +35,6 @@ def __init__(self, platform_id, force_download=False, local_cache=None):
         self.gnps_task_id = None
         self.json_data = None
         self.strains = StrainCollection()
-        self.growth_media = {}
 
         if local_cache is None:
             local_cache = os.path.join(os.getenv('HOME'), 'nplinker_data',
@@ -111,6 +94,8 @@ def __init__(self, platform_id, force_download=False, local_cache=None):
             f.write(str(self.project_json))
 
     def _init_folder_structure(self, local_cache):
+        """Create local cache folders and set up paths for various files"""
+
         # init local cache root
         self.local_cache = local_cache
         self.local_download_cache = os.path.join(self.local_cache, 'downloads')
@@ -150,15 +135,17 @@ def get(self, do_bigscape, extra_bigscape_parameters, use_mibig,
 
         self._download_metabolomics_zipfile(self.gnps_task_id)
 
+        # TODO CG: this function will modify the project_json['genomes'],
+        # this should be done in a better way
         podp_download_and_extract_antismash_data(self.project_json['genomes'],
-                                self.project_download_cache,
-                                self.project_file_cache)
+                                                 self.project_download_cache,
+                                                 self.project_file_cache)
 
         # CG: it extracts strain names and later will be used for strains
         self._parse_genome_labels(self.project_json['genome_metabolome_links'],
                                   self.project_json['genomes'])
 
-        # CG: it generates the strain_mapping.csv file
+        # CG: it generates the strain_mappings.csv file
         self.strains.generate_strain_mappings(
             self.strain_mappings_file,
             os.path.join(self.project_file_cache, 'antismash'))
@@ -211,10 +198,14 @@ def _parse_genome_labels(self, met_records, gen_records):
         # create a set of mappings from one to the other, with the complication that
         # there might be mappings from 2 or more mzXMLs to a single strain.
         # also should record the growth medium using the "sample_preparation_label" field.
+
+        # TODO CG: build mappings from strain id to metabolomics filename (not spectrum id),
+        # when to build spectrum id to metabolomics filename mapping?
         for rec in met_records:
             # this is the global strain identifier we should use
             label = rec['genome_label']
             # only want to record the actual filename of the mzXML URL
+            # TODO CG: is filename always unique?
             filename = os.path.split(rec['metabolomics_file'])[1]
 
             # add the mzXML mapping for this strain
@@ -224,13 +215,11 @@ def _parse_genome_labels(self, met_records, gen_records):
                 temp[label] = [filename]
             mc += 1
 
-            if label in self.growth_media:
-                self.growth_media[label].add(rec['sample_preparation_label'])
-            else:
-                self.growth_media[label] = {rec['sample_preparation_label']}
-
+        # TODO CG: build mappings from strain id to genome id (not BGC id),
+        # when to build BGC id to genome id mapping?
         for rec in gen_records:
             label = rec['genome_label']
+            # TODO CG: change `resolved_id` to `resolved_refseq_id`
             accession = rec.get('resolved_id', None)
             if accession is None:
                 # this will happen for genomes where we couldn't retrieve data or resolve the ID
diff --git a/src/nplinker/scoring/metcalf_scoring.py b/src/nplinker/scoring/metcalf_scoring.py
index 53e6f062..ee029e58 100644
--- a/src/nplinker/scoring/metcalf_scoring.py
+++ b/src/nplinker/scoring/metcalf_scoring.py
@@ -9,17 +9,17 @@
 from nplinker.metabolomics.spectrum import Spectrum
 from nplinker.pickler import load_pickled_data
 from nplinker.pickler import save_pickled_data
-from .linking import DataLinks
-from .linking import isinstance_all
 from .linking import LINK_TYPES
+from .linking import DataLinks
 from .linking import LinkFinder
+from .linking import isinstance_all
 from .methods import ScoringMethod
 from .object_link import ObjectLink
 
 
 if TYPE_CHECKING:
-    from . import LinkCollection
     from ..nplinker import NPLinker
+    from . import LinkCollection
 
 logger = LogConfig.getLogger(__name__)
 
diff --git a/src/nplinker/strain_collection.py b/src/nplinker/strain_collection.py
index 42387c9b..810c5f81 100644
--- a/src/nplinker/strain_collection.py
+++ b/src/nplinker/strain_collection.py
@@ -8,7 +8,6 @@
 from .utils import list_dirs
 from .utils import list_files
 
-
 logger = LogConfig.getLogger(__name__)
 
 


Test patch:
diff --git a/tests/genomics/test_genomics.py b/tests/genomics/test_genomics.py
index dfde9e6c..2abc3dc6 100644
--- a/tests/genomics/test_genomics.py
+++ b/tests/genomics/test_genomics.py
@@ -1,14 +1,43 @@
 from __future__ import annotations
+import json
 import pytest
 from nplinker.genomics import BGC
 from nplinker.genomics import filter_mibig_only_gcf
 from nplinker.genomics import GCF
+from nplinker.genomics import generate_genome_bgc_mappings_file
+from nplinker.genomics import GENOME_BGC_MAPPINGS_FILENAME
 from nplinker.genomics import get_bgcs_from_gcfs
 from nplinker.genomics import get_strains_from_bgcs
 from nplinker.genomics import map_bgc_to_gcf
 from nplinker.genomics import map_strain_to_bgc
 from nplinker.strain_collection import StrainCollection
 from nplinker.strains import Strain
+from .. import DATA_DIR
+
+
+def test_generate_genome_bgc_mappings_file():
+    bgc_dir = DATA_DIR / "antismash"
+
+    generate_genome_bgc_mappings_file(bgc_dir)
+
+    with open(bgc_dir / GENOME_BGC_MAPPINGS_FILENAME) as f:
+        mappings = json.load(f)
+
+    assert mappings["count"] == 2
+
+    assert mappings["mappings"][0]["genome_ID"] == "GCF_000514515.1"
+    assert len(mappings["mappings"][0]["BGC_ID"]) == 20
+    for bgc_id in ["NZ_AZWB01000005.region001", "NZ_KI911412.1.region001"]:
+        assert bgc_id in mappings["mappings"][0]["BGC_ID"]
+    assert "GCF_000514515.1" not in mappings["mappings"][0]["BGC_ID"]
+
+    assert mappings["mappings"][1]["genome_ID"] == "GCF_000514855.1"
+    assert len(mappings["mappings"][1]["BGC_ID"]) == 24
+    for bgc_id in ["NZ_AZWS01000001.region001", "NZ_KI911483.1.region001"]:
+        assert bgc_id in mappings["mappings"][1]["BGC_ID"]
+    assert "GCF_000514855.1" not in mappings["mappings"][1]["BGC_ID"]
+
+    (bgc_dir / GENOME_BGC_MAPPINGS_FILENAME).unlink()
 
 
 @pytest.fixture
@@ -131,6 +160,4 @@ def test_get_strains_from_bgcs(strain_collection, bgc_list,
     map_strain_to_bgc(strain_collection, bgc_list, bgc_genome_mapping)
     strains = get_strains_from_bgcs(bgc_list)
     assert isinstance(strains, StrainCollection)
-    for strain in strain_collection:
-        assert strain in strains
-    # assert strains == strain_collection # use it when issue #113 is solved
+    assert strains == strain_collection