Repository: rmarkello/abagen
Pull request #122 (2019-11-18 20:24:00)

[TEST] Updating CircleCI


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 33eff9c..448b188 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -8,189 +8,164 @@ orbs:
   codecov: codecov/codecov@1.0.5
 
 commands:
-  rununit:
-    description: "Run unit tests for abagen"
+  gettenv:
+    description: "Get/build environment"
     parameters:
       env:
         type: string
+      version:
+        type: string
     steps:
-      - run:
-        no_output_timeout: 40m
-        command: |
-          source activate abagen_<< parameters.env >>
-          mkdir for_testing && cd for_testing
-          cp ../setup.cfg .
-          pytest --cov-report term-missing --cov=abagen --doctest-modules --pyargs abagen
-          mkdir /tmp/src/coverage
-          mv .coverage /tmp/src/coverage/.coverage.py<< parameters.env >>
-
-jobs:
-
-  makeenv_py37:
-    docker:
-      - image: continuumio/miniconda3
-    environment:
-      ABAGEN_DATA: /tmp/src/data
-    working_directory: /tmp/src/abagen
-    steps:
-      - checkout
       - restore_cache:
-          key: conda-py37-v1-{{ checksum "dev_requirements.txt" }}
+          key: conda-<< parameters.env >>-v1-{{ checksum "dev_requirements.txt" }}
       - run:
           name: Generate environment
           command: |
-            if [ ! -d /opt/conda/envs/abagen_py37 ]; then
-              conda create -yq -n abagen_py37 python=3.7
-              source activate abagen_py37
+            if [ ! -d /opt/conda/envs/abagen_<< parameters.env >> ]; then
+              apt-get install -yqq build-essential libsnappy-dev
+              conda create -yq -n abagen_<< parameters.env >> python=<< parameters.version >>
+              source activate abagen_<< parameters.env >>
               pip install -r dev_requirements.txt
             fi
       - save_cache:
-          key: conda-py37-v1-{{ checksum "dev_requirements.txt" }}
+          key: conda-<< parameters.env >>-v1-{{ checksum "dev_requirements.txt" }}
           paths:
-              - /opt/conda/envs/abagen_py37
-     - persist_to_workspace:
+              - /opt/conda/envs/abagen_<< parameters.env >>
+  rununit:
+    description: "Run unit tests for abagen"
+    parameters:
+      env:
+        type: string
+    steps:
+      - run:
+          no_output_timeout: 40m
+          command: |
+            source activate abagen_<< parameters.env >>
+            pip install -e .
+            mkdir for_testing && cd for_testing
+            cp ../setup.cfg .
+            pytest --cov-report term-missing --cov=abagen --doctest-modules --pyargs abagen
+            mkdir /tmp/src/coverage
+            mv .coverage /tmp/src/coverage/.coverage.<< parameters.env >>
+      - persist_to_workspace:
           root: /tmp
           paths:
-              - src/abagen
+              - src/coverage/.coverage.<< parameters.env >>
 
-  fetch_ahba:
+executors:
+  conda_exec:
     docker:
       - image: continuumio/miniconda3
     environment:
       ABAGEN_DATA: /tmp/src/data
     working_directory: /tmp/src/abagen
+
+jobs:
+  makeenv_py37:
+    executor: conda_exec
+    steps:
+      - checkout
+      - gettenv:
+          env: "py37"
+          version: "3.7"
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+              - src/abagen
+  fetch_ahba:
+    executor: conda_exec
     steps:
       - attach_workspace:
           at: /tmp
-      - restore_cache:
-          key: conda-py37-v1-{{ checksum "dev_requirements.txt" }}
+      - gettenv:
+          env: "py37"
+          version: "3.7"
       - run:
           name: Download data
           command: |
-            python -c "import abagen; abagen.fetch_microarray(donors='all')"
-     - persist_to_workspace:
+            pip install -e .
+            python -c "import abagen; abagen.fetch_microarray(donors=['12876', '15496'], n_proc=2)"
+      - persist_to_workspace:
           root: /tmp
           paths:
               - src/data/allenbrain
-
-  unittest_py35:
-    docker:
-      - image: continuumio/miniconda3
-    environment:
-      ABAGEN_DATA: /tmp/src/data
-    working_directory: /tmp/src/abagen
+  style_check:
+    executor: conda_exec
     steps:
       - attach_workspace:
           at: /tmp
-      - restore_cache:
-          key: conda-py35-v1-{{ checksum "dev_requirements.txt" }}
+      - gettenv:
+          env: "py37"
+          version: "3.7"
       - run:
-          name: Generate environment
+          name: Style check
           command: |
-            if [ ! -d /opt/conda/envs/abagen_py35 ]; then
-              conda create -yq -n abagen_py35 python=3.5
-              source activate abagen_py35
-              pip install -r dev_requirements.txt
-              pip install -e .
-            fi
-      - rununit:
-          env: "35"
-      - save_cache:
-          key: conda-py35-v1-{{ checksum "dev_requirements.txt" }}
-          paths:
-              - /opt/conda/envs/abagen_py35
-      - persist_to_workspace:
-          root: /tmp
-          paths:
-              - src/coverage/.coverage.py35
-
-  unittest_py36:
-    docker:
-      - image: continuumio/miniconda3
-    environment:
-      ABAGEN_DATA: /tmp/src/data
-    working_directory: /tmp/src/abagen
+            source activate abagen_py37
+            flake8 abagen
+  build_docs:
+    executor: conda_exec
     steps:
       - attach_workspace:
           at: /tmp
-      - restore_cache:
-          key: conda-py36-v1-{{ checksum "dev_requirements.txt" }}
+      - gettenv:
+          env: "py37"
+          version: "3.7"
       - run:
-          name: Generate environment
+          name: Build documentation
+          no_output_timeout: 40m
           command: |
-            if [ ! -d /opt/conda/envs/abagen_py36 ]; then
-              conda create -yq -n abagen_py36 python=3.6
-              source activate abagen_py36
-              pip install -r dev_requirements.txt
-              pip install -e .
-            fi
+            apt-get install -yqq make
+            source activate abagen_py37
+            pip install -e .
+            # python -c "import abagen; abagen.fetch_microarray(donors='all', n_proc=6)"
+            cd docs/
+            make html  # doctest
+      - store_artifacts:
+          path: /tmp/src/abagen/docs/_build
+  unittest_py35:
+    executor: conda_exec
+    steps:
+      - attach_workspace:
+          at: /tmp
+      - gettenv:
+          env: "py35"
+          version: "3.5"
       - rununit:
-          env: "36"
-      - save_cache:
-          key: conda-py36-v1-{{ checksum "dev_requirements.txt" }}
-          paths:
-              - /opt/conda/envs/abagen_py36
-      - persist_to_workspace:
-          root: /tmp
-          paths:
-              - src/coverage/.coverage.py36
-
-  unittest_py37:  # depends on makeenv_py37
-    docker:
-      - image: continuumio/miniconda3
-    environment:
-      ABAGEN_DATA: /tmp/src/data
-    working_directory: /tmp/src/abagen
+          env: "py35"
+  unittest_py36:
+    executor: conda_exec
     steps:
       - attach_workspace:
           at: /tmp
-      - restore_cache:
-          key: conda-py37-v1-{{ checksum "dev_requirements.txt" }}
-      - run:
-          name: Install abagen
-          command: |
-            pip install -e .
+      - gettenv:
+          env: "py36"
+          version: "3.6"
       - rununit:
-          env: "37"
-      - persist_to_workspace:
-          root: /tmp
-          paths:
-              - src/coverage/.coverage.py37
-
-  style_check:  # depends on makeenv_py37
-    docker:
-      - image: continuumio/miniconda3
-    environment:
-      ABAGEN_DATA: /tmp/src/data
-    working_directory: /tmp/src/abagen
+          env: "py36"
+  unittest_py37:
+    executor: conda_exec
     steps:
       - attach_workspace:
           at: /tmp
-      - restore_cache:
-          key: conda-py37-v1-{{ checksum "dev_requirements.txt" }}
-      - run:
-          name: Style check
-          command: |
-            source activate abagen_py37
-            flake8 abagen
-
-  merge_coverage:   # depends on everything
-    docker:
-      - image: continuumio/miniconda3
-    environment:
-      ABAGEN_DATA: /tmp/src/data
-    working_directory: /tmp/src/abagen
+      - gettenv:
+          env: "py37"
+          version: "3.7"
+      - rununit:
+          env: "py37"
+  merge_coverage:
+    executor: conda_exec
     steps:
       - attach_workspace:
           at: /tmp
-      - restore_cache:
-          key: conda-py37-v1-{{ checksum "dev_requirements.txt" }}
+      - gettenv:
+          env: "py37"
+          version: "3.7"
       - run:
           name: Merge coverage files
           command: |
             apt-get install -yqq curl
             source activate abagen_py37
-            cd /tmp/src/coverage/
+            cd /tmp/src/coverage
             coverage combine
             coverage xml
       - store_artifacts:
@@ -202,24 +177,27 @@ workflows:
   version: 2.1
   build_test:
     jobs:
-      - makeenv_37
+      - makeenv_py37
+      - style_check:
+          requires:
+            - makeenv_py37
       - fetch_ahba:
           requires:
-            - makeenv_37
-      - unittest_35:
+            - makeenv_py37
+      - build_docs:
           requires:
             - fetch_ahba
-      - unittest_36:
+      - unittest_py35:
           requires:
             - fetch_ahba
-      - unittest_37:
+      - unittest_py36:
           requires:
             - fetch_ahba
-      - style_check:
+      - unittest_py37:
           requires:
-            - makeenv_37
+            - fetch_ahba
       - merge_coverage:
           requires:
-            - unittest_35
-            - unittest_36
-            - unittest_37
+            - unittest_py35
+            - unittest_py36
+            - unittest_py37
diff --git a/.travis.yml b/.travis.yml
index 7fab666..478fb53 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,74 +15,17 @@ python:
 
 env:
   matrix:
-    - CHECK_TYPE=linting
     - CHECK_TYPE=docdoctest
-    - CHECK_TYPE=test
-  global:
-    - INSTALL_TYPE=setup
-
-matrix:
-  include:
-  - python: 3.6
-    env:
-      - INSTALL_TYPE=sdist
-      - CHECK_TYPE=test
-  - python: 3.6
-    env:
-      - INSTALL_TYPE=wheel
-      - CHECK_TYPE=test
-  - python: 3.6
-    sudo: required
-    env:
-      - INSTALL_PARQUET=true
-      - CHECK_TYPE=test
 
 before_install:
   - python -m pip install --upgrade pip
-  - if [ "${CHECK_TYPE}" == "linting" ]; then
-      pip install flake8;
-    fi
-  - if [ "${CHECK_TYPE}" == "test" ]; then
-      pip install "pytest>=3.6" pytest-cov coverage coveralls codecov;
-    fi
-  - if [ ! -z "${INSTALL_PARQUET}"]; then
-      sudo apt-get install libsnappy-dev;
-      pip install fastparquet python-snappy;
-    fi
 
 install:
-  - |
-    if [ "${INSTALL_TYPE}" == "setup" ]; then
-      python setup.py install;
-    elif [ "${INSTALL_TYPE}" == "sdist" ]; then
-      python setup.py sdist;
-      pip install dist/*.tar.gz;
-    elif [ "${INSTALL_TYPE}" == "wheel" ]; then
-      python setup.py bdist_wheel;
-      pip install dist/*.whl;
-    else
-      false;
-    fi
+  - python setup.py install;
 
 script:
   - |
-    if [ "${CHECK_TYPE}" == "linting" ]; then
-      flake8 abagen;
-    elif [ "${CHECK_TYPE}" == "docdoctest" ]; then
-      cd docs;
-      pip install -r ./requirements.txt;
-      make html;
-      travis_wait 40 make doctest;
-    elif [ "${CHECK_TYPE}" == "test" ]; then
-      mkdir for_testing && cd for_testing;
-      cp ../setup.cfg .;
-      args="--cov-report term-missing --cov=abagen --doctest-modules --pyargs";
-      travis_wait 40 python -m pytest ${args} ${rootdir} abagen;
-    else
-      false;
-    fi
-
-after_success:
-  - if [ "${CHECK_TYPE}" == "test" ]; then
-      coveralls; codecov;
-    fi
+    cd docs;
+    pip install -r ./requirements.txt;
+    make html;
+    travis_wait 40 make doctest;
diff --git a/abagen/allen.py b/abagen/allen.py
index cc74eeb..af6aa95 100644
--- a/abagen/allen.py
+++ b/abagen/allen.py
@@ -69,7 +69,7 @@ def get_expression_data(atlas, atlas_info=None, *, exact=True,
                         lr_mirror=False, donor_norm='srs',
                         corrected_mni=True, reannotated=True,
                         return_counts=False, return_donors=False,
-                        donors='all', data_dir=None, verbose=1):
+                        donors='all', data_dir=None, verbose=1, n_proc=1):
     """
     Assigns microarray expression data to ROIs defined in `atlas`
 
@@ -193,6 +193,9 @@ def get_expression_data(atlas, atlas_info=None, *, exact=True,
         Specifies verbosity of status messages to display during workflow.
         Higher numbers increase verbosity of messages while zero suppresses all
         messages. Default: 1
+    n_proc : int, optional
+        Number of processors to use to download AHBA data. Can parallelize up
+        to six times. Default: 1
 
     Returns
     -------
@@ -306,7 +309,7 @@ def get_expression_data(atlas, atlas_info=None, *, exact=True,
 
     # fetch files (downloading if necessary) and unpack to variables
     files = datasets.fetch_microarray(data_dir=data_dir, donors=donors,
-                                      verbose=verbose)
+                                      verbose=verbose, n_proc=n_proc)
     microarray = files['microarray']
     annotation = files['annotation']
     pacall = files['pacall']
diff --git a/abagen/datasets/fetchers.py b/abagen/datasets/fetchers.py
index d0f888b..cf70586 100644
--- a/abagen/datasets/fetchers.py
+++ b/abagen/datasets/fetchers.py
@@ -3,6 +3,7 @@
 Functions for downloading the Allen Brain Atlas human microarray dataset.
 """
 
+import multiprocessing as mp
 import os
 from pkg_resources import resource_filename
 
@@ -27,7 +28,7 @@ VALID_DONORS = sorted(WELL_KNOWN_IDS.value_set('subj')
 
 
 def fetch_microarray(data_dir=None, donors=None, resume=True, verbose=1,
-                     convert=True):
+                     convert=True, n_proc=1):
     """
     Downloads the Allen Human Brain Atlas microarray expression dataset
 
@@ -47,6 +48,9 @@ def fetch_microarray(data_dir=None, donors=None, resume=True, verbose=1,
         Whether to convert downloaded CSV files into parquet format for faster
         loading in the future; only available if ``fastparquet`` and ``python-
         snappy`` are installed. Default: True
+    n_proc : int, optional
+        Number of processes to parallelize download if multiple donors are
+        specified. Default: 1
 
     Returns
     -------
@@ -73,6 +77,9 @@ def fetch_microarray(data_dir=None, donors=None, resume=True, verbose=1,
                  'Probes.csv', 'SampleAnnot.csv')
     n_files = len(sub_files)
 
+    if n_proc < 0:
+        n_proc = mp.cpu_count() + n_proc + 1
+
     if donors is None:
         donors = ['12876']
     elif donors == 'all':
@@ -88,16 +95,27 @@ def fetch_microarray(data_dir=None, donors=None, resume=True, verbose=1,
     donors = sorted(set(donors), key=lambda x: donors.index(x))
 
     files = [
-        (os.path.join('normalized_microarray_donor{}'.format(sub), fname),
-         url.format(WELL_KNOWN_IDS.url[sub]),
-         dict(uncompress=True,
-              move=os.path.join('normalized_microarray_donor{}'.format(sub),
-                                'donor{}.zip'.format(sub))))
+        [(os.path.join('normalized_microarray_donor{}'.format(sub), fname),
+            url.format(WELL_KNOWN_IDS.url[sub]),
+            dict(uncompress=True,
+                 move=os.path.join('normalized_microarray_donor{}'.format(sub),
+                                   'donor{}.zip'.format(sub))))
+         for fname in sub_files]
         for sub in donors
-        for fname in sub_files
     ]
 
-    files = _fetch_files(data_dir, files, resume=resume, verbose=verbose)
+    if n_proc > 1:
+        with mp.Pool(n_proc) as pool:
+            results = [pool.apply_async(_fetch_files,
+                                        (data_dir, f),
+                                        dict(resume=resume, verbose=verbose))
+                       for f in files]
+            # flatten outputs into single list
+            files = [l for res in results for l in res.get()]
+    else:
+        # flatten list of lists into single list
+        files = [l for f in files for l in f]
+        files = _fetch_files(data_dir, files, resume=resume, verbose=verbose)
 
     # if we want to convert files to parquet format it's good to do that now
     # this step is _already_ super long, so an extra 1-2 minutes is negligible
diff --git a/docs/conf.py b/docs/conf.py
index bb9e7c9..c41902f 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -116,9 +116,11 @@ intersphinx_mapping = {
 
 doctest_global_setup = """
 import abagen
-abagen.fetch_microarray(donors='all', verbose=0)
+abagen.fetch_microarray(donors='all', verbose=0, n_proc=6)
 atlas = abagen.fetch_desikan_killiany()
 
 import pandas as pd
+pd.set_option('display.width', 1000)
 pd.set_option('max_rows', 6)
+pd.set_option('max_columns', 6)
 """


Test patch:
diff --git a/abagen/tests/conftest.py b/abagen/tests/conftest.py
index eca39b3..ba35d48 100644
--- a/abagen/tests/conftest.py
+++ b/abagen/tests/conftest.py
@@ -10,13 +10,17 @@ from abagen.datasets import fetch_desikan_killiany, fetch_microarray
 
 
 @pytest.fixture(scope='session')
-def datadir():
-    return os.path.join(os.environ['HOME'], 'abagen-data')
+def datadir(tmp_path_factory):
+    dd = os.environ.get('ABAGEN_DATA')
+    if dd is None:
+        dd = str(tmp_path_factory.mktemp('abagen-data'))
+    return dd
 
 
 @pytest.fixture(scope='session')
-def testfiles():
-    return fetch_microarray(donors=['12876', '15496'])
+def testfiles(datadir):
+    return fetch_microarray(data_dir=datadir, donors=['12876', '15496'],
+                            n_proc=2)
 
 
 @pytest.fixture(scope='session')
diff --git a/abagen/tests/datasets/test_fetchers.py b/abagen/tests/datasets/test_fetchers.py
index 46e6e1b..3a9d16a 100644
--- a/abagen/tests/datasets/test_fetchers.py
+++ b/abagen/tests/datasets/test_fetchers.py
@@ -15,6 +15,9 @@ def test_fetch_microarray():
     f3 = fetchers.fetch_microarray(donors='H0351.1009')
     f4 = fetchers.fetch_microarray(donors=None)
 
+    # test n_proc
+    fetchers.fetch_microarray(donors=['12876', '15496'], n_proc=2)
+
     # don't test this -- it will take a wicked long time
     # f5 = datasets.fetch_microarray(donors='all')
 
@@ -36,12 +39,10 @@ def test_fetch_raw_mri():
     f2 = fetchers.fetch_raw_mri(donors='12876')
     f3 = fetchers.fetch_raw_mri(donors='H0351.1009')
     f4 = fetchers.fetch_raw_mri(donors=None)
-    f5 = fetchers.fetch_raw_mri(donors='all')
 
     assert f1 == f2 == f3 == f4
     for k in ['t1w', 't2w']:
         assert len(f1.get(k)) == 1
-        assert len(f5.get(k)) == 6
 
     # check downloading incorrect donor
     with pytest.raises(ValueError):
diff --git a/abagen/tests/datasets/test_utils.py b/abagen/tests/datasets/test_utils.py
index ca5b144..fda14c2 100644
--- a/abagen/tests/datasets/test_utils.py
+++ b/abagen/tests/datasets/test_utils.py
@@ -10,7 +10,7 @@ from abagen.datasets import utils
 
 
 def test_get_dataset_dir(datadir):
-    os.environ.pop('ABAGEN_DATA', None)
+    old = os.environ.pop('ABAGEN_DATA', None)
 
     # check that data dir defaults to $HOME/abagen-data assuming no env var
     expected_base = os.path.expanduser('~/abagen-data')
@@ -38,3 +38,6 @@ def test_get_dataset_dir(datadir):
                                        verbose=0)
     assert data_dir == data_dir2
     shutil.rmtree(data_dir)
+
+    if old is not None:
+        os.environ['ABAGEN_DATA'] = old