Repository: fatiando/verde
Pull request #26 (2018-05-06 08:00:43)

Functions to download sample data 
Adds function `fetch_data` to get the path of a data file from the data 
directory (`$HOME/.verde/data`). If the file is not present, download it
from Github. 
Uses the requests library to download. In the future, this will allow us
to implement chunked downloads of larger files.


Patch:
diff --git a/.travis.yml b/.travis.yml
index ca391cd..d651486 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -49,6 +49,9 @@ matrix:
 
 # Setup the build environment
 before_install:
+    # Copy sample data to the verde data dir to avoid downloading all the time
+    - mkdir -p $HOME/.verde/data
+    - cp data/* $HOME/.verde/data
     # Get the Fatiando CI scripts
     - git clone https://github.com/fatiando/continuous-integration.git
     # Download and install miniconda and setup dependencies
diff --git a/environment.yml b/environment.yml
index 5292f5d..e571e92 100644
--- a/environment.yml
+++ b/environment.yml
@@ -10,6 +10,7 @@ dependencies:
     - pandas
     - xarray
     - scikit-learn
+    - requests
     # Development requirements
     - matplotlib
     - jupyter
diff --git a/requirements.txt b/requirements.txt
index 92d4905..cc6d33d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@ scipy
 pandas
 xarray
 scikit-learn
+requests
 # Development requirements
 matplotlib
 jupyter
diff --git a/verde/datasets/download.py b/verde/datasets/download.py
new file mode 100644
index 0000000..72e81e0
--- /dev/null
+++ b/verde/datasets/download.py
@@ -0,0 +1,44 @@
+"""
+Functions to download, verify, and update a sample dataset.
+"""
+import os
+from warnings import warn
+
+import requests
+
+from ..utils import get_data_dir
+
+
+VERDE_DATA_STORE_URL = 'https://github.com/fatiando/verde/raw/master/data'
+
+
+def fetch_data(filename, force_download=False):
+    """
+    Get the path to a data file in the Verde data directory.
+
+    If it doesn't exist, download it from the remote data store.
+
+    Parameters
+    ----------
+    filename : str
+        The name of the data file to fetch.
+    force_download : bool
+        If True, will download the file even if it already exists.
+
+    Return
+    ------
+    data_path : str
+        A full path to the local file in the data cache directory.
+
+    """
+    data_dir = get_data_dir()
+    data_path = os.path.join(data_dir, filename)
+    if not os.path.exists(data_path) or force_download:
+        data_src = '/'.join([VERDE_DATA_STORE_URL, filename])
+        warn("Downloading data file '{}' from remote data store '{}' to '{}'."
+             .format(filename, VERDE_DATA_STORE_URL, data_dir))
+        response = requests.get(data_src)
+        response.raise_for_status()
+        with open(data_path, 'wb') as dest:
+            dest.write(response.content)
+    return data_path
diff --git a/verde/utils.py b/verde/utils.py
index 37594fa..71c2c72 100644
--- a/verde/utils.py
+++ b/verde/utils.py
@@ -1,10 +1,50 @@
 """
 General utilities for dealing with grids and point generation.
 """
+import os
 import numpy as np
 from sklearn.utils import check_random_state
 
 
+def get_home():
+    """
+    Get the path of the verde home directory.
+
+    Defaults to ``$HOME/.verde``.
+
+    If the folder doesn't already exist, it will be created.
+
+    Returns
+    -------
+    path : str
+        The path of the home directory.
+
+    """
+    home = os.path.abspath(os.environ.get('HOME'))
+    verde_home = os.path.join(home, '.verde')
+    os.makedirs(verde_home, exist_ok=True)
+    return verde_home
+
+
+def get_data_dir():
+    """
+    Get the path of the verde data directory.
+
+    Defaults to ``get_home()/data``.
+
+    If the folder doesn't already exist, it will be created.
+
+    Returns
+    -------
+    path : str
+        The path of the data directory.
+
+    """
+    data_dir = os.path.join(get_home(), 'data')
+    os.makedirs(data_dir, exist_ok=True)
+    return data_dir
+
+
 def check_region(region):
     """
     Check that the given region dimensions are valid.


Test patch:
diff --git a/verde/tests/test_datasets.py b/verde/tests/test_datasets.py
new file mode 100644
index 0000000..1f20873
--- /dev/null
+++ b/verde/tests/test_datasets.py
@@ -0,0 +1,53 @@
+"""
+Test data fetching routines.
+"""
+import os
+import warnings
+from requests.exceptions import HTTPError
+
+import pytest
+
+from ..datasets.download import fetch_data
+
+
+def compare_tiny_data(datapath):
+    """
+    Check if the file exists and compare its content with what we know it
+    should be.
+    """
+    assert os.path.exists(datapath)
+    with open(datapath) as datafile:
+        content = datafile.read()
+    true_content = "\n".join([
+        '# A tiny data file for test purposes only',
+        '1  2  3  4  5  6'])
+    assert content.strip() == true_content
+
+
+# Has to go first in order to guarantee that the file has been downloaded
+def test_fetch_data_from_remote():
+    "Download data from Github to the data directory"
+    with warnings.catch_warnings(record=True) as warn:
+        datapath = fetch_data('tiny-data.txt', force_download=True)
+        assert len(warn) == 1
+        assert issubclass(warn[-1].category, UserWarning)
+        assert str(warn[-1].message).split()[0] == "Downloading"
+    compare_tiny_data(datapath)
+
+
+def test_fetch_data():
+    "Make sure the file exists when not being downloaded"
+    with warnings.catch_warnings(record=True) as warn:
+        datapath = fetch_data('tiny-data.txt')
+        assert not warn
+    compare_tiny_data(datapath)
+
+
+def test_fetch_data_from_store_remote_fail():
+    "Should raise an exception if the remote 404s"
+    with warnings.catch_warnings(record=True) as warn:
+        with pytest.raises(HTTPError):
+            fetch_data('invalid remote file name')
+        assert len(warn) == 1
+        assert issubclass(warn[-1].category, UserWarning)
+        assert str(warn[-1].message).split()[0] == "Downloading"