Repository: PennLINC/qsiprep (language: python)

Issue:
[CI] Check into circleci performance
It looks like some of the TORTOISE workflows are taking almost an hour to run. I'd guess that one of the cpu settings got changed when we switched to pytest


Discussion / hints:

Fix patch:
diff --git a/pyproject.toml b/pyproject.toml
index 67e67628..f6ecf174 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,7 +19,6 @@ requires-python = ">=3.10"
 dependencies = [
     'importlib_resources; python_version < "3.11"',
     "dipy>=1.8.0,<1.9.0",
-    "dmri-amico == 1.5.4",
     "fury",
     "indexed_gzip <= 1.8.7",
     "jinja2 < 3.1",
@@ -32,7 +31,6 @@ dependencies = [
     "numpy <= 1.26.3",
     "pandas < 2.0.0",
     "psutil <= 5.9.8",
-    "pyAFQ == 1.3.2",
     "pybids",
     "pyyaml",
     "scikit-image",
diff --git a/qsiprep/data/boilerplate.bib b/qsiprep/data/boilerplate.bib
index e5da0e10..06b8062a 100644
--- a/qsiprep/data/boilerplate.bib
+++ b/qsiprep/data/boilerplate.bib
@@ -25,26 +25,6 @@ @article{dipy
   publisher={Frontiers}
 }
 
-
-@article{pyafq,
-  title = {Evaluating the reliability of human brain white matter tractometry},
-  author = {Kruper, J. and Yeatman, J.D. and Richie-Halford, A. and Bloom, D. and Grotheer, M. and Caffarra, S. and Kiar, G. and Karipidis, I.I. and Roy, E. and Chandio, B.Q. and Garyfallidis, E. and Rokem, A.},
-  journal = {Aperture Neuro},
-  volume = {1},
-  pages = {1-25},
-  year = {2021},
-}
-
-@article{pyafq2,
-  title = {Tract profiles of White Matter Properties: Automating fiber-tract quantification},
-  volume = {7},
-  doi = {10.1371/journal.pone.0049790},
-  number = {11},
-  journal = {PLoS ONE},
-  author = {Yeatman, Jason D. and Dougherty, Robert F. and Myall, Nathaniel J. and Wandell, Brian A. and Feldman, Heidi M.},
-  year = {2012}
-}
-
 @article{nipype1,
     author = {Gorgolewski, K. and Burns, C. D. and Madison, C. and Clark, D. and Halchenko, Y. O. and Waskom, M. L. and Ghosh, S.},
     doi = {10.3389/fninf.2011.00013},
@@ -556,16 +536,6 @@ @article{noddi
   publisher={Elsevier}
 }
 
-@article{amico,
-  title={Accelerated microstructure imaging via convex optimization (AMICO) from diffusion MRI data},
-  author={Daducci, Alessandro and Canales-Rodr{\'\i}guez, Erick J and Zhang, Hui and Dyrby, Tim B and Alexander, Daniel C and Thiran, Jean-Philippe},
-  journal={NeuroImage},
-  volume={105},
-  pages={32--44},
-  year={2015},
-  publisher={Elsevier}
-}
-
 @article{drbuddi,
   title={DR-BUDDI (Diffeomorphic Registration for Blip-Up blip-Down Diffusion Imaging) method for correcting echo planar imaging distortions},
   author={Irfanoglu, M Okan and Modi, Pooja and Nayak, Amritha and Hutchinson, Elizabeth B and Sarlls, Joelle and Pierpaoli, Carlo},


Test patch:
diff --git a/qsiprep/tests/test_cli.py b/qsiprep/tests/test_cli.py
index d2989568..3505e2c3 100644
--- a/qsiprep/tests/test_cli.py
+++ b/qsiprep/tests/test_cli.py
@@ -2,7 +2,7 @@
 
 import os
 import sys
-from unittest.mock import patch
+from unittest.mock import DEFAULT, patch
 
 import pytest
 from nipype import config as nipype_config
@@ -20,6 +20,8 @@
 
 nipype_config.enable_debug_mode()
 
+DEFAULT_NUM_CPUS = 4
+
 
 @pytest.mark.integration
 @pytest.mark.dsdti_fmap
@@ -181,7 +183,6 @@ def test_drbuddi_rpe(data_dir, output_dir, working_dir):
         "participant",
         f"-w={work_dir}",
         "--sloppy",
-        "--nthreads=4",
         "--anat-modality=none",
         "--denoise-method=none",
         "--b1_biascorrect_stage=none",
@@ -224,7 +225,6 @@ def test_drbuddi_shoreline_epi(data_dir, output_dir, working_dir):
         "--hmc-model=none",
         "--output-resolution=2",
         "--shoreline-iters=1",
-        "--nthreads=1",
     ]
 
     _run_and_generate(TEST_NAME, parameters, test_main=True)
@@ -259,9 +259,8 @@ def test_drbuddi_tensorline_epi(data_dir, output_dir, working_dir):
         "--b1-biascorrect-stage=none",
         "--pepolar-method=DRBUDDI",
         "--hmc-model=tensor",
-        "--output-resolution=2",
+        "--output-resolution=5",
         "--shoreline-iters=1",
-        "--nthreads=1",
     ]
 
     _run_and_generate(TEST_NAME, parameters, test_main=True)
@@ -307,7 +306,6 @@ def test_dscsdsi(data_dir, output_dir, working_dir):
         "--hmc-transform=Rigid",
         "--output-resolution=5",
         "--shoreline-iters=1",
-        "--nthreads=1",
     ]
 
     _run_and_generate(TEST_NAME, parameters, test_main=True)
@@ -350,7 +348,6 @@ def test_dsdti_nofmap(data_dir, output_dir, working_dir):
         "--unringing-method=rpg",
         "--b1-biascorrect-stage=none",
         "--output-resolution=5",
-        "--nthreads=1",
     ]
 
     _run_and_generate(TEST_NAME, parameters, test_main=True)
@@ -393,7 +390,6 @@ def test_dsdti_synfmap(data_dir, output_dir, working_dir):
         "--force-syn",
         "--b1-biascorrect-stage=final",
         "--output-resolution=5",
-        "--nthreads=1",
     ]
 
     _run_and_generate(TEST_NAME, parameters, test_main=True)
@@ -505,8 +501,6 @@ def test_maternal_brain_project(data_dir, output_dir, working_dir):
         "--output-resolution=5",
         "--hmc-model=3dSHORE",
         f"--bids-filter-file={bids_filter}",
-        "--nthreads=4",
-        "--omp-nthreads=4",
     ]
 
     _run_and_generate(TEST_NAME, parameters, test_main=True)
@@ -541,13 +535,34 @@ def test_forrest_gump(data_dir, output_dir, working_dir):
         "--write-graph",
         "--output-resolution=5",
         f"--bids-filter-file={bids_filter}",
-        "--nthreads=4",
-        "--omp-nthreads=4",
     ]
 
     _run_and_generate(TEST_NAME, parameters, test_main=True)
 
 
+def _check_arg_specified(argname, arglist):
+    for arg in arglist:
+        if arg.startswith(argname):
+            return True
+    return False
+
+
+def _update_resources(parameters):
+    """We should use all the available CPUs for testing.
+
+    Sometimes a test will set a specific amount of cpus. In that
+    case, the number should be kept. Otherwise, try to read the
+    env variable (specified in each job in config.yml). If
+    this variable doesn't work, just set it to 4.
+    """
+    nthreads = int(os.environ.get("CIRCLECPUS", DEFAULT_NUM_CPUS))
+    if not _check_arg_specified("--nthreads", parameters):
+        parameters.append(f"--nthreads={nthreads}")
+    if not _check_arg_specified("--omp-nthreads", parameters):
+        parameters.append(f"--omp-nthreads={nthreads}")
+    return parameters
+
+
 def _run_and_generate(test_name, parameters, test_main=True):
     from qsiprep import config
 
@@ -556,6 +571,9 @@ def _run_and_generate(test_name, parameters, test_main=True):
     parameters.append("--notrack")
     parameters.append("-vv")
 
+    # Update resource parameters
+    parameters = _update_resources(parameters)
+
     if test_main:
         # This runs, but for some reason doesn't count toward coverage.
         argv = ["qsiprep"] + parameters