Repository: nilearn/nilearn

Issue:
`make_fresh_openneuro_dataset_urls_index` never tested nor used
we can 
- remove it (but see @kchawla-pi 's [comment](https://github.com/nilearn/nilearn/pull/2514#issuecomment-659010916) )
- have one of the CI test suites run with `boto3` installed


Discussion:
Thx. I think we should have one CI suite with boto3 indeed. 
> Thx. I think we should have one CI suite with boto3 indeed.

Yes. We cannot have code that is not tested.

as it is never used as part of nilearn functionality I think it should be removed. we can keep it in the repo as a separate script if we ever need to re-generate a list of AWS indices
Technically it is tested but never used anywhere in the code base or an example:

https://github.com/nilearn/nilearn/blob/c33cca7adab3d44ce5b2553f65738e6ecf93a3ac/nilearn/datasets/tests/test_utils.py#L451
+1 fore removing that.
So basically putting it as a script in `maint_tools`?
why not remove it ?
I would just remove it. if we ever need something like it in the future, which we probably won't as we don't want to add many dataset fetchers, rewriting these 10 lines will be faster than realizing we have it somewhere and figuring out what this function is or what it does. besides it is made for 1 specific dataset. the test is meaningless -- it just writes a list in a json file then loads it and compares to the original list.
okidoki

Resolving patch:
diff --git a/nilearn/_utils/data_gen.py b/nilearn/_utils/data_gen.py
--- a/nilearn/_utils/data_gen.py
+++ b/nilearn/_utils/data_gen.py
@@ -264,50 +264,6 @@ def generate_labeled_regions(shape,
     return Nifti1Image(data, affine)
 
 
-def generate_labeled_regions_large(shape,
-                                   n_regions,
-                                   random_state=0,
-                                   affine=np.eye(4),
-                                   dtype="int32"):
-    """Similar to generate_labeled_regions, but suitable for a large number of
-    regions.
-
-    See generate_labeled_regions for details.
-
-    Parameters
-    ----------
-    shape : :obj:`tuple` of :obj:`int`
-        Shape of returned array.
-
-    n_regions : :obj:`int`
-        Number of regions to generate.
-
-    random_state : :obj:`int` or :obj:`numpy.random.RandomState` instance, \
-                   optional
-        Random number generator, or seed.
-        Default=0.
-
-    affine : :obj:`numpy.ndarray`, optional
-        Affine of returned image.
-        Default=np.eye(4).
-
-    dtype : :obj:`type`, optional
-        Data type of image.
-        Default='int32'.
-
-    Returns
-    -------
-    Niimg-like object
-        Data has shape "shape", containing region labels.
-
-    """
-    rand_gen = check_random_state(random_state)
-    data = rand_gen.randint(n_regions + 1, size=shape, dtype=dtype)
-    if len(np.unique(data)) != n_regions + 1:
-        raise ValueError("Some labels are missing. Maybe shape is too small.")
-    return Nifti1Image(data, affine)
-
-
 def generate_fake_fmri(shape=(10, 11, 12),
                        length=17,
                        kind="noise",
diff --git a/nilearn/datasets/utils.py b/nilearn/datasets/utils.py
--- a/nilearn/datasets/utils.py
+++ b/nilearn/datasets/utils.py
@@ -855,104 +855,6 @@ def _tree(path, pattern=None, dictionary=False):
     return dirs
 
 
-@fill_doc
-def make_fresh_openneuro_dataset_urls_index(
-        data_dir=None,
-        dataset_version='ds000030_R1.0.4',
-        verbose=1,
-        ):
-    """ONLY intended for Nilearn developers, not general users.
-    Creates a fresh, updated OpenNeuro :term:`BIDS` dataset index from AWS,
-    ready for upload to osf.io .
-
-    Crawls the server where OpenNeuro dataset is stored
-    and makes a JSON file `nistats_fetcher_openneuro_dataset_urls.json'
-    containing a fresh list of dataset file URLs.
-
-    Notes
-    -----
-    Needs Python package `Boto3`.
-
-    Do NOT rename this file.
-
-    This file can now be uploaded to Quick-Files section
-    of the Nilearn account on osf.io .
-
-    Then this file can be downloaded by
-    :func:`datasets.fetch_openneuro_dataset_index`
-
-    Run this function and upload the new file if the URL index downloaded by
-    :func:`datasets.fetch_openneuro_dataset_index` becomes outdated.
-
-    This approach is faster than crawling the servers anew every time
-    the OpenNeuro dataset is downloaded,
-    and circumvents `boto3` as a dependency for everyday use.
-
-    Parameters
-    ----------
-    %(data_dir)s
-    dataset_version : string, optional
-        Dataset version name. Assumes it is of the form [name]_[version].
-        Default is `ds000030_R1.0.4`.
-    %(verbose)s
-
-    Returns
-    -------
-    urls_path : string
-        Path to downloaded dataset index.
-
-    urls : list of string
-        Sorted list of dataset directories.
-
-    """
-    import boto3
-    from botocore.handlers import disable_signing
-    if not data_dir:
-        data_dir = os.path.expanduser('~/Desktop')
-    data_prefix = '{}/{}/uncompressed'.format(
-        dataset_version.split('_')[0], dataset_version)
-
-    data_dir = _get_dataset_dir(data_prefix, data_dir=data_dir,
-                                verbose=verbose)
-
-    # First we download the url list from the uncompressed dataset version
-    urls_path = os.path.join(data_dir,
-                             'nistats_fetcher_openneuro_dataset_urls.json',
-                             )
-    urls = []
-    if os.path.exists(urls_path):
-        with open(urls_path, 'r') as json_file:
-            urls = json.load(json_file)
-        existing_index_msg = ("There is an existing url index at `{}`. "
-                              "Aborting download of fresh index."
-                              .format(urls_path)
-                              )
-        print(existing_index_msg)
-    else:
-        resource = boto3.resource('s3')
-        resource.meta.client.meta.events.register('choose-signer.s3.*',
-                                                  disable_signing)
-        bucket = resource.Bucket('openneuro')
-
-        for obj in bucket.objects.filter(Prefix=data_prefix):
-            # get url of files (keys of directories end with '/')
-            if obj.key[-1] != '/':
-                url = '{}/{}/{}'.format(bucket.meta.client.meta.endpoint_url,
-                                        bucket.name,
-                                        obj.key,
-                                        )
-                urls.append(url)
-        urls = sorted(urls)
-
-        with open(urls_path, 'w') as json_file:
-            json.dump(urls, json_file)
-        print("Saved updated url index to {}.\nUpload it with the same name "
-              "to the quick-files section of osf.io using the Nilearn account "
-              "to update the file without breaking the fetcher download link."
-              .format(urls_path))
-    return urls_path, urls
-
-
 def load_sample_motor_activation_image():
     """Load a single functional image showing motor activations.
 


Test patch:
diff --git a/nilearn/datasets/tests/test_utils.py b/nilearn/datasets/tests/test_utils.py
--- a/nilearn/datasets/tests/test_utils.py
+++ b/nilearn/datasets/tests/test_utils.py
@@ -11,27 +11,16 @@
 import tarfile
 import zipfile
 import urllib
-import json
 from unittest.mock import MagicMock
 from tempfile import mkdtemp, mkstemp
 
-try:
-    import boto3  # noqa: F401
-
-except ImportError:
-    BOTO_INSTALLED = False
-else:
-    BOTO_INSTALLED = True
 
 import numpy as np
 import pytest
 import requests
 
 from nilearn import datasets
-from nilearn.datasets.utils import (
-    _get_dataset_dir, _get_dataset_descr,
-    make_fresh_openneuro_dataset_urls_index
-)
+from nilearn.datasets.utils import _get_dataset_dir, _get_dataset_descr
 from nilearn.datasets import utils
 from nilearn.image import load_img
 
@@ -423,38 +412,6 @@ def test_fetch_files_overwrite(should_cast_path_to_string,
         assert fp.read() == ''
 
 
-@pytest.mark.skipif(not BOTO_INSTALLED,
-                    reason='Boto3  missing; necessary for this test')
-def test_make_fresh_openneuro_dataset_urls_index(tmp_path, request_mocker):
-    dataset_version = 'ds000030_R1.0.4'
-    data_prefix = '{}/{}/uncompressed'.format(
-        dataset_version.split('_')[0], dataset_version)
-    data_dir = _get_dataset_dir(data_prefix, data_dir=str(tmp_path),
-                                verbose=1)
-    url_file = os.path.join(data_dir,
-                            'nistats_fetcher_openneuro_dataset_urls.json',
-                            )
-    # Prepare url files for subject and filter tests
-    file_list = [data_prefix + '/stuff.html',
-                 data_prefix + '/sub-xxx.html',
-                 data_prefix + '/sub-yyy.html',
-                 data_prefix + '/sub-xxx/ses-01_task-rest.txt',
-                 data_prefix + '/sub-xxx/ses-01_task-other.txt',
-                 data_prefix + '/sub-xxx/ses-02_task-rest.txt',
-                 data_prefix + '/sub-xxx/ses-02_task-other.txt',
-                 data_prefix + '/sub-yyy/ses-01.txt',
-                 data_prefix + '/sub-yyy/ses-02.txt']
-    with open(url_file, 'w') as f:
-        json.dump(file_list, f)
-
-    # Only 1 subject and not subject specific files get downloaded
-    datadir, dl_files = make_fresh_openneuro_dataset_urls_index(
-        str(tmp_path), dataset_version)
-    assert isinstance(datadir, str)
-    assert isinstance(dl_files, list)
-    assert len(dl_files) == len(file_list)
-
-
 def test_naive_ftp_adapter():
     sender = utils._NaiveFTPAdapter()
     resp = sender.send(