Repository: cohenlabUNC/clpipe

Issue:
Configuration Defaults Fix
A few changes needed for 1.9:

non-steady-state-outliers* should be default scrubbing
put scrubbing into functional connectivity default
motion outliers default should be true
apply_mask - needs an options example

Discussion:

Patch:
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 0000000..44b7d02
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,25 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.7"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#    - pdf
+#    - epub
+
+
+python:
+   install:
+   - requirements: requirements.txt
\ No newline at end of file
diff --git a/build_clpipe.sh b/build_clpipe.sh
index fd04d00..6d48006 100644
--- a/build_clpipe.sh
+++ b/build_clpipe.sh
@@ -3,7 +3,7 @@
 echo "Starting build"
 
 echo "Clearing previous build"
-rm -r dist/*
+rm -rf dist/*
 
 # Build the .whl file
 echo "Building wheel"
diff --git a/clpipe/config/options.py b/clpipe/config/options.py
index 45c704c..ac51875 100644
--- a/clpipe/config/options.py
+++ b/clpipe/config/options.py
@@ -167,9 +167,9 @@ class Convert2BIDSOptions(Option):
             raise ValidationError("Must be type '.py' or '.json'")
 
     def populate_project_paths(self, project_directory: os.PathLike, source_data: os.PathLike):
-        # create as alt constructor?
+        # Get abs path of project directory
 
-        self.dicom_directory = os.path.abspath(source_data)
+        self.dicom_directory = source_data
         self.conversion_config = os.path.join(project_directory, 'conversion_config.json')
         self.bids_directory = os.path.join(project_directory, 'data_BIDS')
         self.log_directory = os.path.join(project_directory, "logs", "DCM2BIDS_logs")
@@ -711,7 +711,7 @@ class ProjectOptions(Option):
             project_dir (os.PathLike): Root directory of the project.
             source_data (os.PathLike): Directory pointing to the source DICOM data.
         """
-        self.project_directory = os.path.abspath(project_dir)
+        self.project_directory = project_dir
 
         self.convert2bids.populate_project_paths(project_dir, source_data)
         self.bids_validation.populate_project_paths(project_dir)
diff --git a/clpipe/config/package.py b/clpipe/config/package.py
index dec1cfe..4d92dd3 100644
--- a/clpipe/config/package.py
+++ b/clpipe/config/package.py
@@ -1,5 +1,5 @@
 PACKAGE_NAME = 'clpipe'
-VERSION = '1.9.0.3'
+VERSION = '1.9.0.4'
 
 DESCRIPTION = 'clpipe: MRI processing pipeline for high performance clusters'
 REPO_URL = 'https://github.com/cohenlabUNC/clpipe'
@@ -19,18 +19,17 @@ PYTHON_REQUIRES = f'>={PYTHON_VERSION}'
 INSTALL_REQUIRES = [
       'jsonschema==4.17.3',
       'click==8.1.3',
+      'numpy==1.21.6',
       'nipy==0.5.0',
-      'numpy>=1.18.5',
       'pandas==1.3.5',
-      'nibabel>=3',
+      'nibabel==4.0.2',
       'scipy==1.2.2',
-      'sphinx_rtd_theme==1.2.0',
       'psutil==5.9.5',
       'parse==1.19.0',
       'nilearn==0.9.0',
       'dcm2bids==2.1.9',
       'nipype==1.8.6',
-      'pybids>=0.14.0',
+      'pybids>=0.15.6',
       'templateflow==23.0.0',
       "pydantic==1.10.7",
       "matplotlib==3.5.3",
diff --git a/clpipe/project_setup.py b/clpipe/project_setup.py
index 8038780..a5acd98 100644
--- a/clpipe/project_setup.py
+++ b/clpipe/project_setup.py
@@ -23,7 +23,7 @@ class SourceDataError(ValueError):
 
 def project_setup(project_title: str="A Neuroimaging Project", project_dir: os.PathLike=os.getcwd(), 
                   source_data=None, move_source_data=False,
-                  symlink_source_data=False, debug=False):
+                  symlink_source_data=False, debug=False, portable=False):
     """Initialize a clpipe project.
 
     No values can come in as None except source_data.
@@ -38,6 +38,10 @@ def project_setup(project_title: str="A Neuroimaging Project", project_dir: os.P
     # Start up the logger, without file output until we get a project path
     logger = get_logger(STEP_NAME, debug=debug)
 
+    # Ensure project_dir is absolute if portability not requested
+    if not portable:
+        project_dir = os.path.abspath(project_dir)
+
     # Create a default DICOM dir for source if no source given
     default_dicom_dir = os.path.join(project_dir, DEFAULT_DICOM_DIR)
     
diff --git a/clpipe/source.py b/clpipe/source.py
index 73af927..9dd2758 100644
--- a/clpipe/source.py
+++ b/clpipe/source.py
@@ -1,76 +1,61 @@
-from .config_json_parser import ClpipeConfigParser
+from .config.options import ProjectOptions
 from .utils import get_logger
 import os
 from pathlib import Path
 
 from .batch_manager import BatchManager, Job
 
+STEP_NAME = "flywheel_sync"
 FLYWHEEL_TEMP_DIR_NAME = "'<TemporaryDirectory '\\'''/"
 
 def flywheel_sync(config_file, source_url=None, dropoff_dir=None, temp_dir=None, submit=False, debug=False):
     """Sync your project's DICOMs with Flywheel."""
     
-    config_parser = ClpipeConfigParser(config_file)
-    config = config_parser.config
+    options: ProjectOptions = ProjectOptions.load(config_file)
+    options.source.load_cli_args(
+        dropoff_directory=dropoff_dir,
+        temp_directory=temp_dir,
+        source_url=source_url,
+    )
 
-    if not dropoff_dir:
-        dropoff_dir = config["SourceOptions"]["DropoffDirectory"]
+    logger = get_logger(STEP_NAME, debug=debug, log_dir=options.get_logs_dir())
 
-    if not source_url:
-        source_url = config["SourceOptions"]["SourceURL"]
+    if options.source.temp_directory == "":
+        options.source.temp_directory = Path(options.project_directory) / "flywheel_tmp"
+        logger.debug(f"Using default temp directory: {options.source.temp_directory}")
 
-    if not temp_dir:
-        temp_dir = config["SourceOptions"]["TempDirectory"]
-        if temp_dir == "":
-            temp_dir = Path(config["ProjectDirectory"]) / "flywheel_tmp"
-    temp_dir = Path(temp_dir)
+    if not Path(options.source.temp_directory).exists():
+        Path(options.source.temp_directory).mkdir(parents=True)
 
-    try:
-        cmd_line_opts = config['SourceOptions']['CommandLineOpts']
-    except KeyError:
-        cmd_line_opts = ""
-
-    if not temp_dir.exists():
-        temp_dir.mkdir(parents=True)
-        
-    batch_config = config['BatchConfig']
-    mem_usage = config['SourceOptions']['MemUsage']
-    time_usage = config['SourceOptions']['TimeUsage']
-    n_threads = config['SourceOptions']['CoreUsage']
-
-    log_dir = Path(config["ProjectDirectory"]) / "logs" / "sync_logs"
+    log_dir = Path(options.project_directory) / "logs" / "sync_logs"
     if not log_dir.exists:
         logger.debug(f"Creating log dir: {log_dir}")
         log_dir.mkdir()
 
-    batch_manager = BatchManager(batch_config, log_dir, debug=debug)
+    batch_manager = BatchManager(options.batch_config_path, log_dir, debug=debug)
     batch_manager.create_submission_head()
-    batch_manager.update_mem_usage(mem_usage)
-    batch_manager.update_time(time_usage)
-    batch_manager.update_nthreads(n_threads)
-
-    logger = get_logger("flywheel_sync", debug=debug)
+    batch_manager.update_mem_usage(options.source.mem_usage)
+    batch_manager.update_time(options.source.time_usage)
+    batch_manager.update_nthreads(options.source.core_usage)
 
-    logger.debug(f"Using sync dir: {dropoff_dir}")
-    logger.debug(f"Using source URL: {source_url}")
-    logger.debug(f"Using temp directory: {temp_dir}")
-    if cmd_line_opts != "":
-        logger.debug(f"Added additional Flywheel options: {cmd_line_opts}")
+    logger.debug(f"Using dropoff directory: {options.source.dropoff_directory}")
+    logger.debug(f"Using source URL: {options.source.source_url}")
+    logger.debug(f"Using temp directory: {options.source.temp_directory}")
+    if options.source.commandline_opts != "":
+        logger.debug(f"Added additional Flywheel options: {options.source.commandline_opts}")
 
     flywheel_generated_temp_dir = os.path.join(os.getcwd(), FLYWHEEL_TEMP_DIR_NAME)
-
     logger.debug(f"Temporary Directory: {flywheel_generated_temp_dir}")
     
-    submission_string = f"fw sync --include dicom --tmp-path {temp_dir} {cmd_line_opts} {source_url} {dropoff_dir}; rm -r {flywheel_generated_temp_dir}"
+    submission_string = f"fw sync --include dicom --tmp-path {options.source.temp_directory} {options.source.commandline_opts} {options.source.source_url} {options.source.dropoff_directory}; rm -r {flywheel_generated_temp_dir}"
     job_id = f"flywheel_sync_DICOM"
 
     job = Job(job_id, submission_string)
 
     batch_manager.addjob(job)
-    
     batch_manager.compile_job_strings()
 
     if submit:
         batch_manager.submit_jobs()
     else:
-        batch_manager.print_jobs()
\ No newline at end of file
+        batch_manager.print_jobs()
diff --git a/docs/overview.rst b/docs/overview.rst
index 67e237c..c63dd1e 100644
--- a/docs/overview.rst
+++ b/docs/overview.rst
@@ -19,9 +19,34 @@ command. To create addition configuration files for your dataset, use the follow
 This command will create a default configuration file with whatever name you specified. 
 The top of the default configuration file looks like this:
 
-.. literalinclude:: ../clpipe/data/defaultConfig.json
-   :language: json
-   :lines: 1-20
+.. code-block:: json
+
+    {
+        "project_title": "test_project",
+        "contributors": "SET CONTRIBUTORS",
+        "project_directory": "/nas/longleaf/home/user/clpipe",
+        "email_address": "SET EMAIL ADDRESS",
+        "source": {
+            "source_url": "fw://",
+            "dropoff_directory": "",
+            "temp_directory": "",
+            "commandline_opts": "-y",
+            "time_usage": "1:0:0",
+            "mem_usage": "10G",
+            "core_usage": "2"
+        },
+        "convert2bids": {
+            "dicom_directory": "/nas/longleaf/home/user/clpipe/data_DICOMs",
+            "bids_directory": "/nas/longleaf/home/user/clpipe/data_BIDS",
+            "conversion_config": "/nas/longleaf/home/user/clpipe/conversion_config.json",
+            "dicom_format_string": "",
+            "time_usage": "1:0:0",
+            "mem_usage": "5000",
+            "core_usage": "2",
+            "log_directory": "/nas/longleaf/home/user/clpipe/logs/DCM2BIDS_logs"
+        },
+        ...
+    }
 
 The configuration file consists of some project-level metadata, such as "ProjectTitle",
 and a set of Option blocks that contain their own sub-settings. Each Option block
diff --git a/docs/postprocessing.rst b/docs/postprocessing.rst
index 9cf5832..296a286 100644
--- a/docs/postprocessing.rst
+++ b/docs/postprocessing.rst
@@ -48,47 +48,54 @@ Step Options section for more information about configuring this section.
 
 .. code-block:: json
 
-	"PostProcessingOptions2": {
-		"WorkingDirectory": "",
-		"WriteProcessGraph": true,
-		"TargetImageSpace": "MNI152NLin2009cAsym",
-		"TargetTasks": [],
-		"TargetAcquisitions": [],
-		"ProcessingSteps": [
+	"postprocessing": {
+		"working_directory": "/nas/longleaf/home/user/work",
+		"write_process_graph": true,
+		"target_directory": "/nas/longleaf/home/clpipe/data_fmriprep",
+		"target_image_space": "MNI152NLin2009cAsym",
+		"target_tasks": [],
+		"target_acquisitions": [],
+		"output_directory": "/nas/longleaf/home/user/clpipe/data_postprocess",
+		"processing_steps": [
 			"SpatialSmoothing",
 			"TemporalFiltering",
-			"IntensityNormalization"
+			"IntensityNormalization",
+			"ApplyMask"
 		],
-		"ProcessingStepOptions": {
-			"TemporalFiltering": {
-				"Implementation":"fslmaths",
-				"FilteringHighPass": 0.008,
-				"FilteringLowPass": -1,
-				"FilteringOrder": 2
-			}, 
+		"processing_step_options": {
+			"temporal_filtering": {
+				"implementation": "fslmaths",
+				"filtering_high_pass": 0.008,
+				"filtering_low_pass": -1,
+				"filtering_order": 2
+			},
 			...additional processing step options
 		},
-		"ConfoundOptions": {
-			"Columns": [
-				"csf", "csf_derivative1", "white_matter", "white_matter_derivative1"
+		"confound_options": {
+			"columns": [
+				"csf",
+				"csf_derivative1",
+				"white_matter",
+				"white_matter_derivative1"
 			],
-			"MotionOutliers": {
-				"Include": true,
-				"ScrubVar": "framewise_displacement",
-				"Threshold": 0.9,
-				"ScrubAhead": 0,
-				"ScrubBehind": 0,
-				"ScrubContiguous": 0
+			"motion_outliers": {
+				"include": true,
+				"scrub_var": "framewise_displacement",
+				"threshold": 0.9,
+				"scrub_ahead": 0,
+				"scrub_behind": 0,
+				"scrub_contiguous": 0
 			}
 		},
-		"BatchOptions": {
-			"MemoryUsage": "20000",
-			"TimeUsage": "2:0:0",
-			"NThreads": "1"
-    	}	
-	}
+		"batch_options": {
+			"memory_usage": "20G",
+			"time_usage": "2:0:0",
+			"n_threads": "1"
+		},
+		"log_directory": "/nas/longleaf/home/user/clpipe/logs/postprocess_logs"
 
 **Top-Level Definitions**
+
 .. autoclass:: clpipe.config.options.PostProcessingOptions
 	:members:
 
@@ -353,10 +360,12 @@ while this step adds a variable number of columns to your confounds.
 
 Resample
 --------------------
+
 .. autoclass:: clpipe.config.project.Resample
 
 Trim Timepoints
 --------------------
+
 .. autoclass:: clpipe.config.project.TrimTimepoints
 
 Batch Options
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 0000000..e69de29
diff --git a/requirements.txt b/requirements.txt
index b18e205..d66c327 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,14 +1,16 @@
+# Development Tools
 build
 pytest
 wheel
 
-# Documentation builder tools
+# Documentation Dependencies
 markdown-it-py==2.0.1 
 myst-parser==0.16.1 
 sphinx==4.5.0
 sphinx-jsonschema==1.17.2
 sphinx-click==4.3.0
+sphinx_rtd_theme==1.2.0
 
-# Install this package in edit mode
+# Install clpipe in edit mode
 -e .
 


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 90470b1..54f6370 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -14,6 +14,14 @@ def test_cli():
 
     assert e.value.code == 0
 
+def test_source_cli(clpipe_dicom_dir: Path):
+    """Simple test to ensure the source commmand runs without error."""
+
+    runner = CliRunner()
+    result = runner.invoke(flywheel_sync_cli, ["-c", str(clpipe_dicom_dir / "clpipe_config.json")])
+    print(result.output)
+    assert result.exit_code == 0
+
 def test_project_setup_cli(scatch_dir: Path):
     """Check basic functionality."""
     
diff --git a/tests/test_project_setup.py b/tests/test_project_setup.py
index 76409d5..b732c9d 100644
--- a/tests/test_project_setup.py
+++ b/tests/test_project_setup.py
@@ -4,6 +4,7 @@ import os
 from typing import List
 
 from clpipe.project_setup import project_setup, SourceDataError
+from clpipe.config.options import ProjectOptions
 
 PROJECT_TITLE = "test_project"
 
@@ -12,10 +13,21 @@ def test_project_setup_no_source(tmp_path: Path):
     """Check that clpipe creates an empty data_DICOMs folder in the project
     directory when no source is provided.
     """
-
     project_setup(project_title=PROJECT_TITLE, project_dir=tmp_path)
 
     assert Path(tmp_path / "data_DICOMs").exists()
+    assert Path(tmp_path / "clpipe_config.json").exists()
+
+def test_project_setup_no_source_rel_path(tmp_path: Path):
+    """Assure proper paths created in config when relative path provided."""
+    os.chdir(tmp_path)
+
+    project_setup(project_title=PROJECT_TITLE, project_dir=".")
+
+    config: ProjectOptions = ProjectOptions().load(Path(tmp_path / "clpipe_config.json"))
+
+    # Ensure absolute path was applied to config
+    assert(not config.fmriprep.output_directory.startswith("."))
 
 
 def test_project_setup_referenced_source(tmp_path: Path, source_data: Path):
diff --git a/tests/test_source.py b/tests/test_source.py
new file mode 100644
index 0000000..2f113a2
--- /dev/null
+++ b/tests/test_source.py
@@ -0,0 +1,7 @@
+from clpipe.source import flywheel_sync
+from pathlib import Path
+
+
+def test_flywheel_sync(clpipe_dicom_dir):
+    """Test to ensure flywheel sync is working."""
+    flywheel_sync(clpipe_dicom_dir / "clpipe_config.json", submit=False, debug=True)
\ No newline at end of file