Repository: ME-ICA/tedana

Issue:
What to do if a provided ICA mixing matrix doesn't result in likely BOLD components
### Summary

In the August 2023 dev call (#957) @BahmanTahayori  commented that he had trouble tracking down a bug because it was not obvious that the ICA was re-run. The suggestion was to make a re-run of ICA more obvious in the outputs.

### Additional Detail

When ICA is rerun the following warning message is added to the log file: https://github.com/ME-ICA/tedana/blob/f9daaa01cbe3280f4ffb343f5751392d3b2df89d/tedana/workflows/tedana.py#L680-L681

In writing this up, I'm realizing we might want an error rather than a more prominent warning. If a mixing matrix is calculated within tedana and doesn't converge or doesn't generate likely BOLD components, then reseeding and rerunning ICA an under-the-hood decision. We log when this happens, but it's not a major issue to highlight to users. The real issue here is when there is a user provided a mixing matrix and the output of tedana does not use that mixing matrix. It might make sense to end execution rather than output something that's different from what the user requested.

### Next Steps

- Decide if this should be an error or a warning
  - `Error: User-provided mixing matrix did not result in likely BOLD components. Rerun with a different user-provided mixing matrix or run ICA within tedana where ICA will be recalculated if no likely BOLD components are found`
  - If we want a more prominent warning, what should be be?
    - An added line in the log `WARNING: User-provided ICA mixing matrix did not result in likely BOLD components. tedana is recalculating ICA`
    - A red line of text at the top of `tedana_report.html`

Discussion:
@tsalo: I like the error idea.@handwerkerd: I'll also add to this, now that we're using RobustICA, if we don't get components, it doesn't make sense to re-run RobustICA

Solution patch:
diff --git a/tedana/workflows/tedana.py b/tedana/workflows/tedana.py
index 7ebe468..0502930 100644
--- a/tedana/workflows/tedana.py
+++ b/tedana/workflows/tedana.py
@@ -877,35 +877,31 @@ def tedana_workflow(
                 n_independent_echos=n_independent_echos,
             )
             n_likely_bold_comps = selector.n_likely_bold_comps_
-            LGR.info("Selecting components from ICA results")
-            selector = selection.automatic_selection(
-                component_table,
-                selector,
-                n_echos=n_echos,
-                n_vols=n_vols,
-                n_independent_echos=n_independent_echos,
-            )
-            n_likely_bold_comps = selector.n_likely_bold_comps_
-            if (n_restarts < maxrestart) and (n_likely_bold_comps == 0):
-                LGR.warning("No BOLD components found. Re-attempting ICA.")
-            elif n_likely_bold_comps == 0:
-                LGR.warning("No BOLD components found, but maximum number of restarts reached.")
-                keep_restarting = False
+
+            if n_likely_bold_comps == 0:
+                if ica_method.lower() == "robustica":
+                    LGR.warning("No BOLD components found with robustICA mixing matrix.")
+                    keep_restarting = False
+                elif n_restarts >= maxrestart:
+                    LGR.warning(
+                        "No BOLD components found, but maximum number of restarts reached."
+                    )
+                    keep_restarting = False
+                else:
+                    LGR.warning("No BOLD components found. Re-attempting ICA.")
+                    # If we're going to restart, temporarily allow force overwrite
+                    io_generator.overwrite = True
+                    # Create a re-initialized selector object if rerunning
+                    # Since external_regressor_config might have been expanded to remove
+                    # regular expressions immediately after initialization,
+                    # store and copy this key
+                    tmp_external_regressor_config = selector.tree["external_regressor_config"]
+                    selector = ComponentSelector(tree)
+                    selector.tree["external_regressor_config"] = tmp_external_regressor_config
+                    RepLGR.disabled = True  # Disable the report to avoid duplicate text
             else:
                 keep_restarting = False
 
-            # If we're going to restart, temporarily allow force overwrite
-            if keep_restarting:
-                io_generator.overwrite = True
-                # Create a re-initialized selector object if rerunning
-                # Since external_regressor_config might have been expanded to remove
-                # regular expressions immediately after initialization,
-                # store and copy this key
-                tmp_external_regressor_config = selector.tree["external_regressor_config"]
-                selector = ComponentSelector(tree, out_dir)
-                selector.tree["external_regressor_config"] = tmp_external_regressor_config
-
-            RepLGR.disabled = True  # Disable the report to avoid duplicate text
         RepLGR.disabled = False  # Re-enable the report after the while loop is escaped
         io_generator.overwrite = overwrite  # Re-enable original overwrite behavior
     else:
@@ -939,6 +935,12 @@ def tedana_workflow(
             n_independent_echos=n_independent_echos,
         )
 
+        if selector.n_likely_bold_comps_ == 0:
+            LGR.warning("No BOLD components found with user-provided ICA mixing matrix.")
+
+    # TODO The ICA mixing matrix should be written out after it is created
+    #     It is currently being written after component selection is done
+    #     and rewritten if an existing mixing matrix is given as an input
     comp_names = component_table["Component"].values
     mixing_df = pd.DataFrame(data=mixing, columns=comp_names)
     io_generator.save_file(mixing_df, "ICA mixing tsv")


Test patch:
diff --git a/tedana/tests/data/cornell_three_echo_verbose_outputs.txt b/tedana/tests/data/cornell_three_echo_verbose_outputs.txt
new file mode 100644
index 0000000..3008e42
--- /dev/null
+++ b/tedana/tests/data/cornell_three_echo_verbose_outputs.txt
@@ -0,0 +1,146 @@
+S0map.nii.gz
+T2starmap.nii.gz
+dataset_description.json
+desc-ICAAccepted_components.nii.gz
+desc-ICAAccepted_stat-z_components.nii.gz
+desc-ICA_components.nii.gz
+desc-ICA_decomposition.json
+desc-tedana_metrics.json
+desc-tedana_metrics.tsv
+desc-tedana_registry.json
+desc-ICACrossComponent_metrics.json
+desc-ICA_status_table.tsv
+desc-ICA_decision_tree.json
+desc-ICA_mixing.tsv
+desc-ICA_stat-z_components.nii.gz
+desc-PCACrossComponent_metrics.json
+desc-PCA_decomposition.json
+desc-PCA_metrics.json
+desc-PCA_metrics.tsv
+desc-PCA_mixing.tsv
+desc-PCA_stat-z_components.nii.gz
+desc-adaptiveGoodSignal_mask.nii.gz
+desc-denoised_bold.nii.gz
+desc-optcom_bold.nii.gz
+desc-confounds_timeseries.tsv
+desc-rmse_statmap.nii.gz
+desc-ICAAveragingWeights_components.nii.gz
+desc-ICAS0_stat-F_statmap.nii.gz
+desc-ICAT2_stat-F_statmap.nii.gz
+desc-PCAAveragingWeights_components.nii.gz
+desc-PCAS0_stat-F_statmap.nii.gz
+desc-PCAT2_stat-F_statmap.nii.gz
+desc-limited_S0map.nii.gz
+desc-limited_T2starmap.nii.gz
+desc-optcomRejected_bold.nii.gz
+desc-optcom_whitened_bold.nii.gz
+echo-1_desc-Denoised_bold.nii.gz
+echo-1_desc-ICAS0ModelPredictions_components.nii.gz
+echo-1_desc-ICAT2ModelPredictions_components.nii.gz
+echo-1_desc-ICA_components.nii.gz
+echo-1_desc-PCAS0ModelPredictions_components.nii.gz
+echo-1_desc-PCAT2ModelPredictions_components.nii.gz
+echo-1_desc-PCA_components.nii.gz
+echo-1_desc-Rejected_bold.nii.gz
+echo-2_desc-Denoised_bold.nii.gz
+echo-2_desc-ICAS0ModelPredictions_components.nii.gz
+echo-2_desc-ICAT2ModelPredictions_components.nii.gz
+echo-2_desc-ICA_components.nii.gz
+echo-2_desc-PCAS0ModelPredictions_components.nii.gz
+echo-2_desc-PCAT2ModelPredictions_components.nii.gz
+echo-2_desc-PCA_components.nii.gz
+echo-2_desc-Rejected_bold.nii.gz
+echo-3_desc-Denoised_bold.nii.gz
+echo-3_desc-ICAS0ModelPredictions_components.nii.gz
+echo-3_desc-ICAT2ModelPredictions_components.nii.gz
+echo-3_desc-ICA_components.nii.gz
+echo-3_desc-PCAS0ModelPredictions_components.nii.gz
+echo-3_desc-PCAT2ModelPredictions_components.nii.gz
+echo-3_desc-PCA_components.nii.gz
+echo-3_desc-Rejected_bold.nii.gz
+figures
+figures/adaptive_mask.svg
+figures/carpet_optcom.svg
+figures/carpet_denoised.svg
+figures/carpet_accepted.svg
+figures/carpet_rejected.svg
+figures/comp_000.png
+figures/comp_001.png
+figures/comp_002.png
+figures/comp_003.png
+figures/comp_004.png
+figures/comp_005.png
+figures/comp_006.png
+figures/comp_007.png
+figures/comp_008.png
+figures/comp_009.png
+figures/comp_010.png
+figures/comp_011.png
+figures/comp_012.png
+figures/comp_013.png
+figures/comp_014.png
+figures/comp_015.png
+figures/comp_016.png
+figures/comp_017.png
+figures/comp_018.png
+figures/comp_019.png
+figures/comp_020.png
+figures/comp_021.png
+figures/comp_022.png
+figures/comp_023.png
+figures/comp_024.png
+figures/comp_025.png
+figures/comp_026.png
+figures/comp_027.png
+figures/comp_028.png
+figures/comp_029.png
+figures/comp_030.png
+figures/comp_031.png
+figures/comp_032.png
+figures/comp_033.png
+figures/comp_034.png
+figures/comp_035.png
+figures/comp_036.png
+figures/comp_037.png
+figures/comp_038.png
+figures/comp_039.png
+figures/comp_040.png
+figures/comp_041.png
+figures/comp_042.png
+figures/comp_043.png
+figures/comp_044.png
+figures/comp_045.png
+figures/comp_046.png
+figures/comp_047.png
+figures/comp_048.png
+figures/comp_049.png
+figures/comp_050.png
+figures/comp_051.png
+figures/comp_052.png
+figures/comp_053.png
+figures/comp_054.png
+figures/comp_055.png
+figures/comp_056.png
+figures/comp_057.png
+figures/comp_058.png
+figures/comp_059.png
+figures/comp_060.png
+figures/comp_061.png
+figures/comp_062.png
+figures/comp_063.png
+figures/comp_064.png
+figures/comp_065.png
+figures/comp_066.png
+figures/comp_067.png
+figures/comp_068.png
+figures/pca_criteria.png
+figures/pca_variance_explained.png
+figures/rmse_brain.svg
+figures/rmse_timeseries.svg
+figures/s0_brain.svg
+figures/s0_histogram.svg
+figures/t2star_brain.svg
+figures/t2star_histogram.svg
+references.bib
+report.txt
+tedana_report.html
diff --git a/tedana/tests/data/reject_all_tree.json b/tedana/tests/data/reject_all_tree.json
new file mode 100644
index 0000000..a4b71ec
--- /dev/null
+++ b/tedana/tests/data/reject_all_tree.json
@@ -0,0 +1,20 @@
+{
+    "tree_id": "reject_all_tree",
+    "info": "designed to reject all components to test functionality",
+    "report": "If you are publishing results using this tree, you are doing something wrong. ",
+    "necessary_metrics": [],
+    "intermediate_classifications": [],
+    "classification_tags": ["Rejected just because"],
+    "nodes": [
+        {
+            "functionname": "manual_classify",
+            "parameters": {"new_classification": "rejected", "decide_comps": "all"},
+            "kwargs": {
+                "clear_classification_tags": true,
+                "dont_warn_reclassify": true,
+                "tag": "Rejected just because"
+            },
+            "_comment": "All components are labeled as 'rejected'."
+        }
+    ]
+}
diff --git a/tedana/tests/test_external_metrics.py b/tedana/tests/test_external_metrics.py
index 25177cb..e8fcdb0 100644
--- a/tedana/tests/test_external_metrics.py
+++ b/tedana/tests/test_external_metrics.py
@@ -126,8 +126,8 @@ def sample_mixing_matrix():
 
     return pd.read_csv(
         op.join(
-            THIS_DIR,
-            "../../.testing_data_cache/three-echo/TED.three-echo/desc_ICA_mixing_static.tsv",
+            data_for_testing_info("path"),
+            "three-echo/TED.three-echo/desc_ICA_mixing_static.tsv",
         ),
         delimiter="\t",
     ).to_numpy()
diff --git a/tedana/tests/test_integration.py b/tedana/tests/test_integration.py
index 3dafa9c..4572898 100644
--- a/tedana/tests/test_integration.py
+++ b/tedana/tests/test_integration.py
@@ -24,7 +24,14 @@ LOGGER = logging.getLogger(__name__)
 TestLGR = logging.getLogger("TESTING")
 
 
-def check_integration_outputs(fname, outpath, n_logs=1):
+def check_integration_outputs(
+    fname,
+    outpath,
+    n_logs=1,
+    max_expected_comp=None,
+    extra_expected_files=None,
+    unexpected_files=None,
+):
     """
     Checks outputs of integration tests.
 
@@ -34,6 +41,30 @@ def check_integration_outputs(fname, outpath, n_logs=1):
         Path to file with expected outputs
     outpath : str
         Path to output directory generated from integration tests
+    n_logs : int
+        Number of expected tedana_date.tsv logs to expect in the output directory
+        Default is 1, but can be 2 if a test is designed to run twice in the same directory
+    max_expected_comp : int or None
+        The maximum component number as shown in the expected file comp_xxx.png
+        Default is None and will expect to find the exact file names in fname
+        If this is an integer it will add or remove comp_xxx.png file names
+        until it matches max_expected_comp.
+        This is useful for avoiding needing to create a new template file for
+        fname just because an integration test outputs a different total
+        number of components than another test
+        Can also be set to -1 and then the count of component figures will be ignored.
+        This is useful when a test uses robustic with a low number for n-robust-runs
+        A few iterations of robustICA tests all functionality, but makes it possible
+        for different iterations to result in a variable final number of components.
+        Setting this to -1 should only be used in that situation.
+    extra_expected_files: list[str]
+        A list of files that are in outpath, but are not in fname.
+        Whenever this is used in a test, it is recommended have a comment to explain why
+        the files are not expected to be present. Default is None
+    unexpected_files : list[str]
+        A list of files that is in the list of expected outputs in fname, but are not expected
+        Whenever this is used in a test, it is recommended have a comment to explain why
+        the files are not expected to be present. Default is None
     """
 
     # Gets filepaths generated by integration test
@@ -56,6 +87,38 @@ def check_integration_outputs(fname, outpath, n_logs=1):
         expected_files = f.read().splitlines()
     expected_files = [os.path.normpath(path) for path in expected_files]
 
+    if extra_expected_files:
+        for append_name in extra_expected_files:
+            expected_files.append(append_name)
+
+    comp_fig_regex = "^figures/.*comp_[0-9]{3}.png$"
+    if max_expected_comp and max_expected_comp > 0:
+        tmp_comps = sorted([out for out in expected_files if re.match(comp_fig_regex, out)])
+        stored_max_expected_comp = int(tmp_comps[-1][-7:-4])
+        # Since the prefix can be a variable length, pull from an existing file
+        comp_prefix = tmp_comps[1][:-7]
+        # if there are more components than expected, then delete extra file names
+        if max_expected_comp < stored_max_expected_comp:
+            for comp_idx in range(max_expected_comp + 1, stored_max_expected_comp + 1):
+                expected_files.remove(f"{comp_prefix}{comp_idx:03}.png")
+        # if there are fewer components than expected, then add expected file names
+        if stored_max_expected_comp < max_expected_comp:
+            for comp_idx in range(stored_max_expected_comp + 1, max_expected_comp + 1):
+                expected_files.append(f"{comp_prefix}{comp_idx:03}.png")
+
+    if max_expected_comp == -1:
+        # Remove all component figures from comparisions in both expected_files and found_files
+        remove_fig_names = [out for out in expected_files if re.match(comp_fig_regex, out)]
+        for remove_name in remove_fig_names:
+            expected_files.remove(remove_name)
+        remove_fig_names = [out for out in found_files if re.match(comp_fig_regex, out)]
+        for remove_name in remove_fig_names:
+            found_files.remove(remove_name)
+
+    if unexpected_files:
+        for remove_name in unexpected_files:
+            expected_files.remove(remove_name)
+
     if sorted(found_files) != sorted(expected_files):
         expected_not_found = sorted(list(set(expected_files) - set(found_files)))
         found_not_expected = sorted(list(set(found_files) - set(expected_files)))
@@ -150,7 +213,9 @@ def test_integration_five_echo(skip_integration):
 
     # compare the generated output files
     fn = resource_filename("tedana", "tests/data/nih_five_echo_outputs_verbose.txt")
-    check_integration_outputs(fn, out_dir)
+    # Since robustica with only 4 iterations might result in a variable number of finale comps,
+    #  using max_expected_comp=-1 to not check the number of component files.
+    check_integration_outputs(fn, out_dir, max_expected_comp=-1)
 
 
 def test_integration_four_echo(skip_integration):
@@ -212,34 +277,20 @@ def test_integration_four_echo(skip_integration):
 
 
 def test_integration_three_echo(skip_integration):
-    """Integration test of the full tedana workflow using three-echo test data."""
+    """Integration test of the full tedana workflow with the CLI using three-echo test data."""
 
     if skip_integration:
         pytest.skip("Skipping three-echo integration test")
 
     test_data_path, osf_id = data_for_testing_info("three-echo")
     out_dir = os.path.abspath(os.path.join(test_data_path, "../../outputs/three-echo"))
-    out_dir_manual = f"{out_dir}-rerun"
 
     if os.path.exists(out_dir):
         shutil.rmtree(out_dir)
 
-    if os.path.exists(out_dir_manual):
-        shutil.rmtree(out_dir_manual)
-
     # download data and run the test
     download_test_data(osf_id, test_data_path)
-    tedana_cli.tedana_workflow(
-        data=f"{test_data_path}/three_echo_Cornell_zcat.nii.gz",
-        tes=[14.5, 38.5, 62.5],
-        out_dir=out_dir,
-        low_mem=True,
-        tedpca="aic",
-    )
 
-    # Test re-running, but use the CLI
-    # TODO Move this to a separate integration test, use the fixed desc_ICA_mixing_static.tsv that
-    #      is distributed with the testing data, and test specific outputs for consistent values
     args = [
         "-d",
         f"{test_data_path}/three_echo_Cornell_zcat.nii.gz",
@@ -248,20 +299,106 @@ def test_integration_three_echo(skip_integration):
         "38.5",
         "62.5",
         "--out-dir",
-        out_dir_manual,
-        "--debug",
-        "--verbose",
-        "-f",
-        "--mix",
-        os.path.join(out_dir, "desc-ICA_mixing.tsv"),
-        "--t2smap",
-        os.path.join(out_dir, "T2starmap.nii.gz"),
+        out_dir,
+        "--lowmem",
+        "--tedpca",
+        "mdl",
     ]
     tedana_cli._main(args)
 
     # compare the generated output files
     fn = resource_filename("tedana", "tests/data/cornell_three_echo_outputs.txt")
-    check_integration_outputs(fn, out_dir)
+    check_integration_outputs(
+        fn, out_dir, max_expected_comp=64, extra_expected_files=["tedana_call.sh"]
+    )
+
+
+def test_integration_three_echo_noacc_rerun(skip_integration, caplog):
+    """Integration test three-echo test data, runninga supplied mixing matrix, no accepted."""
+
+    if skip_integration:
+        pytest.skip("Skipping three-echo integration test")
+
+    test_data_path, osf_id = data_for_testing_info("three-echo")
+    download_test_data(osf_id, test_data_path)
+
+    out_dir = os.path.abspath(os.path.join(test_data_path, "../../outputs/three-echo-rerun"))
+
+    if os.path.exists(out_dir):
+        shutil.rmtree(out_dir)
+
+    # Run first with robustICA and no accepted components
+    # Using tedpca=n_components to speed up processing
+    n_components = 55
+    tedana_cli.tedana_workflow(
+        data=f"{test_data_path}/three_echo_Cornell_zcat.nii.gz",
+        tes=[14.5, 38.5, 62.5],
+        out_dir=out_dir,
+        ica_method="robustica",
+        tedpca=n_components,
+        n_robust_runs=4,
+        tree=resource_filename("tedana", "tests/data/reject_all_tree.json"),
+        verbose=True,
+    )
+
+    # compare the generated output files
+    fn = resource_filename("tedana", "tests/data/cornell_three_echo_verbose_outputs.txt")
+    # Files for accepted components are not expected because all components are rejected
+    # Files for PCA component estimation are not expected because a fixed # of comps was specified
+    unexpected_files = [
+        "desc-ICAAccepted_components.nii.gz",
+        "desc-ICAAccepted_stat-z_components.nii.gz",
+        "desc-PCACrossComponent_metrics.json",
+        "figures/pca_criteria.png",
+        "figures/pca_variance_explained.png",
+    ]
+    # Since robustica with only 4 iterations might result in a variable number of finale comps,
+    #  using max_expected_comp=-1 to not check the number of component files.
+    check_integration_outputs(fn, out_dir, max_expected_comp=-1, unexpected_files=unexpected_files)
+    assert "No BOLD components found with robustICA mixing matrix" in caplog.text
+
+    # Rerun in the same dir with supplied mixing matrix & tree that results in no accepted comps
+    # This is also the test that supplies a t2smap
+    # and can be used to confirm T2* and S0 QC image are correctly not rendered in tedana_report
+    tedana_cli.tedana_workflow(
+        data=f"{test_data_path}/three_echo_Cornell_zcat.nii.gz",
+        tes=[14.5, 38.5, 62.5],
+        out_dir=out_dir,
+        debug=True,
+        mixing_file=os.path.join(test_data_path, "desc_ICA_mixing_static.tsv"),
+        overwrite=True,
+        tree=resource_filename("tedana", "tests/data/reject_all_tree.json"),
+        t2smap=os.path.join(out_dir, "T2starmap.nii.gz"),
+    )
+
+    # compare the generated output files. There are 68 components in the predefined mixing matrix
+    fn = resource_filename("tedana", "tests/data/cornell_three_echo_verbose_outputs.txt")
+    # The since using a mixing matrix and skipping PCA,
+    # the same files from the previous iteration are excluded
+    # Also excluding counfounds, which is generated with the T2* map estimate,
+    # which is skipped here
+    unexpected_files.append("desc-confounds_timeseries.tsv")
+    # mixing_static is the name of the provided mixing matrix
+    # When rerun in the same directory, report_old is a backup of the immediately previous report
+    extra_expected_files = ["desc_ICA_mixing_static.tsv", "report_old.txt"]
+    check_integration_outputs(
+        fn,
+        out_dir,
+        max_expected_comp=68,
+        n_logs=2,
+        extra_expected_files=extra_expected_files,
+        unexpected_files=unexpected_files,
+    )
+    assert "No BOLD components found with user-provided ICA mixing matrix." in caplog.text
+
+    # TODO When a test is rerun in the same directory, overwrite, only overwrite existing files.
+    #      Decide if it should also delete existing files from a previous iteration.
+    #      In this test, the files from the first verbose run are retained even though the
+    #      second run is not verbose.
+    #      That is what is being tested here because that is the current functionality.
+    #      If the functionality of overwrite is changed, the second run of tedana in this test
+    #      should then exclude the verbose files.
+    # TODO Add tests for the numerical outputs of the results with the user-provided mixing matrix
 
 
 def test_integration_three_echo_external_regressors_single_model(skip_integration, caplog):
@@ -302,12 +439,18 @@ def test_integration_three_echo_external_regressors_single_model(skip_integratio
             "tedana", "tests/data/external_regress_Ftest_3echo.tsv"
         ),
         low_mem=True,
-        tedpca="aic",
+        tedpca=0.9,
     )
 
     # compare the generated output files
     fn = resource_filename("tedana", "tests/data/cornell_three_echo_outputs.txt")
-    check_integration_outputs(fn, out_dir)
+    # Files for PCA component estimation are not expected because a fixed % variance was specified
+    unexpected_files = [
+        "desc-PCACrossComponent_metrics.json",
+        "figures/pca_criteria.png",
+        "figures/pca_variance_explained.png",
+    ]
+    check_integration_outputs(fn, out_dir, max_expected_comp=42, unexpected_files=unexpected_files)
 
     assert "It is strongly recommended to provide an external mask," in caplog.text
 
@@ -342,7 +485,6 @@ def test_integration_three_echo_external_regressors_motion_task_models(skip_inte
         ),
         mixing_file=f"{test_data_path}/desc_ICA_mixing_static.tsv",
         low_mem=True,
-        tedpca="aic",
     )
 
     # compare the generated output files
@@ -357,6 +499,7 @@ def test_integration_reclassify_insufficient_args(skip_integration):
     guarantee_reclassify_data()
 
     test_data_path, osf_id = data_for_testing_info("three-echo")
+    download_test_data(osf_id, test_data_path)
     out_dir = os.path.abspath(
         os.path.join(test_data_path, "../../outputs/reclassify/insufficient")
     )
diff --git a/tedana/tests/test_workflows_parser_utils.py b/tedana/tests/test_workflows_parser_utils.py
index 40f70da..3fb61ee 100644
--- a/tedana/tests/test_workflows_parser_utils.py
+++ b/tedana/tests/test_workflows_parser_utils.py
@@ -6,7 +6,7 @@ import os.path as op
 import pandas as pd
 import pytest
 
-from tedana.workflows import tedana as tedana_cli
+from tedana.tests.utils import data_for_testing_info
 from tedana.workflows.parser_utils import (
     check_tedpca_value,
     parse_manual_list_int,
@@ -73,7 +73,7 @@ def test_parse_manual_list_int():
     assert tmp == []
 
     to_accept = [i for i in range(3)]
-    test_data_path = op.abspath(op.join(tedana_cli.__file__, "../../../.testing_data_cache"))
+    test_data_path = data_for_testing_info("path")
     acc_df = pd.DataFrame(data=to_accept, columns=["Components"])
     acc_csv_fname = op.join(test_data_path, "accept.csv")
     acc_df.to_csv(acc_csv_fname)