Repository: nilearn/nilearn

Issue:
Add Netmats Megatrawl dataset
http://humanconnectome.org/about/pressroom/project-news/netmats-megatrawl-released/



Discussion:
Totally, that would be awesome!

I just browsed quickly and I see no way to get the data. Did you mean that you want the parceallations to be downloaded and the study re-run?

I guess that what you wanted were the correlation matrices.

> I just browsed quickly and I see no way to get the data. Did you mean that you
> want the parceallations to be downloaded and the study re-run?

I couldn't find it either, but Steve Smith claims it's easy. For a
certain definition of easy I guess.

There's supposed to be a download link. If we can't find it, we'll have
to ask Steve.

> I guess that what you wanted were the correlation matrices.

Yes

Found it. It is really not obvious. Select any run (I take 3T_Q1-Q6related468_MSMsulc_d25_ts3).
Click on "nodes+edges 2" to get the netjs graph.
Sniff a little bit, find that "main.js" do the job.
Open it: https://db.humanconnectome.org/megatrawl/3T_Q1-Q6related468_MSMsulc_d25_ts2/netjs/js/main.js
Find the matrices, eg : https://db.humanconnectome.org/megatrawl/3T_Q1-Q6related468_MSMsulc_d25_ts2/netjs/data/dataset1/Znet2.txt

The files are very light, we can totally include them directly into nilearn. However, they have a user agreement that we must include.

Most of the work will be to understand dataformat and load it in a user-friendly fashion.

I let anybody motivated take it from here ;).

Sidenote: the netjs visualization if very pretty and they basically did most of the work to adapt it to our usecase. It shouldn't be too much work to generate it from our data. PR welcome in `nilearn_sandbox` ;).

> I let anybody motivated take it from here ;)

What about me ? I can try to address this. If not, I announce it here.

I'm OK with you working on it but you already have a lot of battles to fight.


Resolving patch:
diff --git a/examples/manipulating_visualizing/plot_visualize_megatrawls_netmats.py b/examples/manipulating_visualizing/plot_visualize_megatrawls_netmats.py
new file mode 100644
--- /dev/null
+++ b/examples/manipulating_visualizing/plot_visualize_megatrawls_netmats.py
@@ -0,0 +1,34 @@
+"""
+Visualizing Megatrawls Network Matrices from Human Connectome Project
+=====================================================================
+
+This example shows how to fetch network matrices data from HCP beta-release
+of the Functional Connectivity Megatrawl project.
+
+See :func:`nilearn.datasets.fetch_megatrawls_netmats` documentation for more details.
+"""
+################################################################################
+# Fetching the partial correlation matrices of dimensionality d=300 with
+# timeseries method 'eigen regression' by importing datasets module
+from nilearn import datasets
+
+netmats = datasets.fetch_megatrawls_netmats(dimensionality=300,
+                                            timeseries='eigen_regression',
+                                            matrices='partial_correlation')
+# Partial correlation matrices array of size (300, 300) are stored in the name
+# of 'correlation_matrices'
+partial_correlation = netmats.correlation_matrices
+
+################################################################################
+# Visualization
+# Importing matplotlib and nilearn plotting modules to use its utilities for
+# plotting correlation matrices
+import matplotlib.pyplot as plt
+from nilearn import plotting
+
+title = "Partial correlation matrices of d=300 with timeseries='eigen_regression'"
+plt.figure()
+plt.imshow(partial_correlation, interpolation="nearest", cmap=plotting.cm.bwr)
+plt.colorbar()
+plt.title(title)
+plt.show()
diff --git a/nilearn/datasets/__init__.py b/nilearn/datasets/__init__.py
--- a/nilearn/datasets/__init__.py
+++ b/nilearn/datasets/__init__.py
@@ -6,7 +6,8 @@
 from .func import (fetch_haxby_simple, fetch_haxby, fetch_nyu_rest,
                    fetch_adhd, fetch_miyawaki2008,
                    fetch_localizer_contrasts, fetch_abide_pcp,
-                   fetch_localizer_calculation_task, fetch_mixed_gambles)
+                   fetch_localizer_calculation_task, fetch_mixed_gambles,
+                   fetch_megatrawls_netmats)
 from .atlas import (fetch_atlas_craddock_2012, fetch_atlas_destrieux_2009,
                     fetch_atlas_harvard_oxford, fetch_atlas_msdl,
                     fetch_atlas_power_2011, fetch_atlas_smith_2009,
@@ -19,4 +20,5 @@
            'fetch_atlas_craddock_2012', 'fetch_atlas_destrieux_2009',
            'fetch_atlas_harvard_oxford', 'fetch_atlas_msdl',
            'fetch_atlas_power_2011', 'fetch_atlas_smith_2009',
-           'fetch_atlas_yeo_2011', 'fetch_mixed_gambles', 'fetch_atlas_aal']
+           'fetch_atlas_yeo_2011', 'fetch_mixed_gambles', 'fetch_atlas_aal',
+           'fetch_megatrawls_netmats']
diff --git a/nilearn/datasets/func.py b/nilearn/datasets/func.py
--- a/nilearn/datasets/func.py
+++ b/nilearn/datasets/func.py
@@ -12,6 +12,7 @@
                     _read_md5_sum_file, _tree, _filter_columns)
 
 from .._utils.compat import BytesIO, _basestring, _urllib
+from .._utils.numpy_conversions import csv_to_array
 
 
 def fetch_haxby_simple(data_dir=None, url=None, resume=True, verbose=1):
@@ -459,7 +460,7 @@ def fetch_adhd(n_subjects=None, data_dir=None, url=None, resume=True,
     phenotypic = _fetch_files(data_dir, [phenotypic], resume=resume,
                               verbose=verbose)[0]
 
-    ## Load the csv file
+    # Load the csv file
     phenotypic = np.genfromtxt(phenotypic, names=True, delimiter=',',
                                dtype=None)
 
@@ -1279,3 +1280,129 @@ def fetch_mixed_gambles(n_subjects=1, data_dir=None, url=None, resume=True,
         X, y, mask_img = _load_mixed_gambles(map(nibabel.load, data.zmaps))
         data.zmaps, data.gain, data.mask_img = X, y, mask_img
     return data
+
+
+def fetch_megatrawls_netmats(dimensionality=100, timeseries='eigen_regression',
+                             matrices='partial_correlation', data_dir=None,
+                             resume=True, verbose=1):
+    """Downloads and returns Network Matrices data from MegaTrawls release in HCP.
+
+    This data can be used to predict relationships between imaging data and
+    non-imaging behavioural measures such as age, sex, education, etc.
+    The network matrices are estimated from functional connectivity
+    datasets of 461 subjects. Full technical details in [1] [2].
+
+    .. versionadded:: 0.2.2
+
+    Parameters
+    ----------
+    dimensionality: int, optional
+        Valid inputs are 25, 50, 100, 200, 300. By default, network matrices
+        estimated using Group ICA brain parcellations of 100 components/dimensions
+        will be returned.
+
+    timeseries: str, optional
+        Valid inputs are 'multiple_spatial_regression' or 'eigen_regression'. By
+        default 'eigen_regression', matrices estimated using first principal
+        eigen component timeseries signals extracted from each subject data
+        parcellations will be returned. Otherwise, 'multiple_spatial_regression'
+        matrices estimated using spatial regressor based timeseries signals
+        extracted from each subject data parcellations will be returned.
+
+    matrices: str, optional
+        Valid inputs are 'full_correlation' or 'partial_correlation'. By default,
+        partial correlation matrices will be returned otherwise if selected
+        full correlation matrices will be returned.
+
+    data_dir: str, default is None, optional
+        Path of the data directory. Used to force data storage in a specified
+        location.
+
+    resume: bool, default is True
+        This parameter is required if a partially downloaded file is needed
+        to be resumed to download again.
+
+    verbose: int, default is 1
+        This parameter is used to set the verbosity level to print the message
+        to give information about the processing.
+        0 indicates no information will be given.
+
+    Returns
+    -------
+    data: Bunch
+        dictionary-like object, the attributes are :
+
+        - 'dimensions': int, consists of given input in dimensions.
+
+        - 'timeseries': str, consists of given input in timeseries method.
+
+        - 'matrices': str, consists of given type of specific matrices.
+
+        - 'correlation_matrices': ndarray, consists of correlation matrices
+          based on given type of matrices. Array size will depend on given
+          dimensions (n, n).
+        - 'description': data description
+
+    References
+    ----------
+    [1] Stephen Smith et al, HCP beta-release of the Functional Connectivity
+        MegaTrawl.
+        April 2015 "HCP500-MegaTrawl" release.
+        https://db.humanconnectome.org/megatrawl/
+
+    [2] Smith, S.M. et al. Nat. Neurosci. 18, 1565-1567 (2015).
+
+    [3] N.Filippini, et al. Distinct patterns of brain activity in young
+        carriers of the APOE-e4 allele.
+        Proc Natl Acad Sci USA (PNAS), 106::7209-7214, 2009.
+
+    [4] S.Smith, et al. Methods for network modelling from high quality rfMRI data.
+        Meeting of the Organization for Human Brain Mapping. 2014
+
+    [5] J.X. O'Reilly et al. Distinct and overlapping functional zones in the
+        cerebellum defined by resting state functional connectivity.
+        Cerebral Cortex, 2009.
+
+    Note: See description for terms & conditions on data usage.
+
+    """
+    url = "http://www.nitrc.org/frs/download.php/8037/Megatrawls.tgz"
+    opts = {'uncompress': True}
+
+    error_message = "Invalid {0} input is provided: {1}, choose one of them {2}"
+    # standard dataset terms
+    dimensionalities = [25, 50, 100, 200, 300]
+    if dimensionality not in dimensionalities:
+        raise ValueError(error_message.format('dimensionality', dimensionality,
+                                              dimensionalities))
+    timeseries_methods = ['multiple_spatial_regression', 'eigen_regression']
+    if timeseries not in timeseries_methods:
+        raise ValueError(error_message.format('timeseries', timeseries,
+                                              timeseries_methods))
+    output_matrices_names = ['full_correlation', 'partial_correlation']
+    if matrices not in output_matrices_names:
+        raise ValueError(error_message.format('matrices', matrices,
+                                              output_matrices_names))
+
+    dataset_name = 'Megatrawls'
+    data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir, verbose=verbose)
+    description = _get_dataset_descr(dataset_name)
+
+    timeseries_map = dict(multiple_spatial_regression='ts2', eigen_regression='ts3')
+    matrices_map = dict(full_correlation='Znet1.txt', partial_correlation='Znet2.txt')
+    filepath = [(os.path.join(
+        '3T_Q1-Q6related468_MSMsulc_d%d_%s' % (dimensionality, timeseries_map[timeseries]),
+        matrices_map[matrices]), url, opts)]
+
+    # Fetch all the files
+    files = _fetch_files(data_dir, filepath, resume=resume, verbose=verbose)
+
+    # Load the files into arrays
+    correlation_matrices = csv_to_array(files[0])
+
+    return Bunch(
+        dimensions=dimensionality,
+        timeseries=timeseries,
+        matrices=matrices,
+        correlation_matrices=correlation_matrices,
+        description=description)


Test patch:
diff --git a/nilearn/datasets/tests/test_func.py b/nilearn/datasets/tests/test_func.py
--- a/nilearn/datasets/tests/test_func.py
+++ b/nilearn/datasets/tests/test_func.py
@@ -16,6 +16,7 @@
 from . import test_utils as tst
 
 from nilearn.datasets import utils, func
+from nilearn._utils.testing import assert_raises_regex
 
 from nilearn._utils.compat import _basestring, _urllib
 
@@ -362,3 +363,68 @@ def test_fetch_mixed_gambles():
         assert_equal(mgambles["zmaps"][0], os.path.join(datasetdir, "zmaps",
                                                         "sub001_zmaps.nii.gz"))
         assert_equal(len(mgambles["zmaps"]), n_subjects)
+
+
+def test_check_parameters_megatrawls_datasets():
+    # testing whether the function raises the same error message
+    # if invalid input parameters are provided
+    message = "Invalid {0} input is provided: {1}."
+
+    for invalid_input_dim in [1, 5, 30]:
+        assert_raises_regex(ValueError,
+                            message.format('dimensionality', invalid_input_dim),
+                            func.fetch_megatrawls_netmats,
+                            dimensionality=invalid_input_dim)
+
+    for invalid_input_timeserie in ['asdf', 'time', 'st2']:
+        assert_raises_regex(ValueError,
+                            message.format('timeseries', invalid_input_timeserie),
+                            func.fetch_megatrawls_netmats,
+                            timeseries=invalid_input_timeserie)
+
+    for invalid_output_name in ['net1', 'net2']:
+        assert_raises_regex(ValueError,
+                            message.format('matrices', invalid_output_name),
+                            func.fetch_megatrawls_netmats,
+                            matrices=invalid_output_name)
+
+
+@with_setup(tst.setup_tmpdata, tst.teardown_tmpdata)
+def test_fetch_megatrawls_netmats():
+    # smoke test to see that files are fetched and read properly
+    # since we are loading data present in it
+    files_dir = os.path.join(tst.tmpdir, 'Megatrawls', '3T_Q1-Q6related468_MSMsulc_d100_ts3')
+    os.makedirs(files_dir)
+    with open(os.path.join(files_dir, 'Znet2.txt'), 'w') as net_file:
+        net_file.write("1")
+
+    files_dir2 = os.path.join(tst.tmpdir, 'Megatrawls', '3T_Q1-Q6related468_MSMsulc_d300_ts2')
+    os.makedirs(files_dir2)
+    with open(os.path.join(files_dir2, 'Znet1.txt'), 'w') as net_file2:
+        net_file2.write("1")
+
+    megatrawl_netmats_data = func.fetch_megatrawls_netmats(data_dir=tst.tmpdir)
+
+    # expected number of returns in output name should be equal
+    assert_equal(len(megatrawl_netmats_data), 5)
+    # check if returned bunch should not be empty
+    # dimensions
+    assert_not_equal(megatrawl_netmats_data.dimensions, '')
+    # timeseries
+    assert_not_equal(megatrawl_netmats_data.timeseries, '')
+    # matrices
+    assert_not_equal(megatrawl_netmats_data.matrices, '')
+    # correlation matrices
+    assert_not_equal(megatrawl_netmats_data.correlation_matrices, '')
+    # description
+    assert_not_equal(megatrawl_netmats_data.description, '')
+
+    # check if input provided for dimensions, timeseries, matrices to be same
+    # to user settings
+    netmats_data = func.fetch_megatrawls_netmats(data_dir=tst.tmpdir,
+                                                 dimensionality=300,
+                                                 timeseries='multiple_spatial_regression',
+                                                 matrices='full_correlation')
+    assert_equal(netmats_data.dimensions, 300)
+    assert_equal(netmats_data.timeseries, 'multiple_spatial_regression')
+    assert_equal(netmats_data.matrices, 'full_correlation')
diff --git a/nilearn/tests/test_param_validation.py b/nilearn/tests/test_param_validation.py
--- a/nilearn/tests/test_param_validation.py
+++ b/nilearn/tests/test_param_validation.py
@@ -2,7 +2,6 @@
 Test the _utils.param_validation module
 """
 
-import warnings
 import numpy as np
 
 from nose.tools import assert_true, assert_equal