Repository: weecology/retriever

Issue:
change test setup
Currently tests  are run from the test directory 
`os.chdir(os.path.dirname(os.path.abspath(__file__)))`
we need the to run in the retriever directory

```
Ref:https://github.com/weecology/retriever/pull/626
My initial thought was to not cd into test in the first place. 
We are only doing this because the data for DelMoral and AvianBodySize is stored in test/raw_data. If as part of setup_module we copy ./test/raw_data to ./raw_data the data should be accessible. 
So we would remove the os.chdir(os.path.dirname(os.path.abspath(__file__))) line and replace it with something a system call that does the raw_data copying. 
The teardown_module looks like it's probably handling the cleanup of that already.

```



Discussion:

Resolving patch:
diff --git a/__init__.py b/__init__.py
--- a/__init__.py
+++ b/__init__.py
@@ -44,6 +44,7 @@
             print("The Retriever lacks permission to access the ~/.retriever/ directory.")
             raise
 SCRIPT_SEARCH_PATHS = [
+    "./",
     'scripts',
     os.path.join(HOME_DIR, 'scripts/')
 ]
diff --git a/lib/compile.py b/lib/compile.py
--- a/lib/compile.py
+++ b/lib/compile.py
@@ -141,12 +141,12 @@ def get_value(key):
 
 
 def add_dialect(table_dict, table):
-    '''
+    """
     Reads dialect key of JSON script and extracts key-value pairs to store them
     in python script
 
     Contains properties such 'nulls', delimiter', etc
-    '''
+    """
     for (key, val) in table['dialect'].items():
         # dialect related key-value pairs
         # copied as is
@@ -161,13 +161,13 @@ def add_dialect(table_dict, table):
 
 
 def add_schema(table_dict, table):
-    '''
+    """
     Reads schema key of JSON script and extracts values to store them in
     python script
 
     Contains properties related to table schema, such as 'fields' and cross-tab
     column name ('ct_column').
-    '''
+    """
     for (key, val) in table['schema'].items():
         # schema related key-value pairs
 
@@ -178,14 +178,14 @@ def add_schema(table_dict, table):
             column_list = []
             for obj in val:
                 # fields is a collection of JSON objects
-                #(similar to a list of dicts in python)
+                # (similar to a list of dicts in python)
 
                 if "size" in obj:
                     column_list.append((obj["name"],
                                         (obj["type"], obj["size"])))
                 else:
                     column_list.append((obj["name"],
-                                        (obj["type"], )))
+                                        (obj["type"],)))
 
             table_dict["columns"] = column_list
 
@@ -197,13 +197,17 @@ def add_schema(table_dict, table):
 
 
 def compile_json(json_file):
-    '''
+    """
     Function to compile JSON script files to python scripts
     The scripts are created with `retriever create_json <script_name` using
     command line
-    '''
+    """
     json_object = json.load(open(json_file + ".json", "r"))
 
+    if "retriever" not in json_object.keys():
+        # Compile only files that have retriever key
+        return
+
     values = {}
     values['urls'] = {}
 
@@ -233,7 +237,7 @@ def compile_json(json_file):
             # Array of table objects
             tables = {}
             for table in value:
-                 # Maintain a dict for table keys and values
+                # Maintain a dict for table keys and values
                 table_dict = {}
 
                 try:
diff --git a/lib/parse_script_to_json.py b/lib/parse_script_to_json.py
--- a/lib/parse_script_to_json.py
+++ b/lib/parse_script_to_json.py
@@ -15,6 +15,7 @@ def parse_script_to_json(script_file, location=SCRIPT_DIR):
     replace = []
     keys_to_ignore = ["template"]
     urls = {}
+    values["retriever"] = "True"
 
     for line in [str(line).strip() for line in definition]:
         if line and ':' in line and not line[0] == '#':


Test patch:
diff --git a/test/test_integration.py b/test/test_integration.py
--- a/test/test_integration.py
+++ b/test/test_integration.py
@@ -3,15 +3,14 @@
 import imp
 import os
 import shutil
-import pytest
 
+import pytest
 from retriever.lib.compile import compile_json
 from retriever.lib.parse_script_to_json import parse_script_to_json
 from retriever import HOME_DIR, ENGINE_LIST
 from retriever.lib.tools import file_2string
 from retriever.lib.tools import create_file
 
-
 simple_csv = {'name': 'simple_csv',
               'raw_data': "a,b,c\n1,2,3\n4,5,6\n",
               'script': "shortname: simple_csv\ntable: simple_csv, http://example.com/simple_csv.txt",
@@ -28,25 +27,27 @@
             'expect_out': 'a,b,c,val\n1,1,c1,1.1\n1,1,c2,1.2\n1,2,c1,2.1\n1,2,c2,2.2\n'}
 
 autopk_crosstab = {'name': 'autopk_crosstab',
-            'raw_data': "a,b,c1,c2\n1,1,1.1,1.2\n1,2,2.1,2.2\n",
-            'script': "shortname: autopk_crosstab\ntable: autopk_crosstab, http://example.com/autopk_crosstab.txt\n*column: record_id, pk-auto\n*column: a, int\n*column: b, int\n*ct_column: c\n*column: val, ct-double\n*ct_names: c1,c2",
-            'expect_out': 'record_id,a,b,c,val\n1,1,1,c1,1.1\n2,1,1,c2,1.2\n3,1,2,c1,2.1\n4,1,2,c2,2.2\n'}
+                   'raw_data': "a,b,c1,c2\n1,1,1.1,1.2\n1,2,2.1,2.2\n",
+                   'script': "shortname: autopk_crosstab\ntable: autopk_crosstab, http://example.com/autopk_crosstab.txt\n*column: record_id, pk-auto\n*column: a, int\n*column: b, int\n*ct_column: c\n*column: val, ct-double\n*ct_names: c1,c2",
+                   'expect_out': 'record_id,a,b,c,val\n1,1,1,c1,1.1\n2,1,1,c2,1.2\n3,1,2,c1,2.1\n4,1,2,c2,2.2\n'}
 
 skip_csv = {'name': 'skip_csv',
-              'raw_data': "a,b,c\n1,2,3\n4,5,6\n",
-              'script': "shortname: skip_csv\ntable: skip_csv, http://example.com/skip_csv.txt\n*do_not_bulk_insert: True\n*column: a, skip\n*column: b, int\n*column: c, int",
-              'expect_out': 'b,c\n2,3\n5,6\n'}
+            'raw_data': "a,b,c\n1,2,3\n4,5,6\n",
+            'script': "shortname: skip_csv\ntable: skip_csv, http://example.com/skip_csv.txt\n*do_not_bulk_insert: True\n*column: a, skip\n*column: b, int\n*column: c, int",
+            'expect_out': 'b,c\n2,3\n5,6\n'}
 
 extra_newline = {'name': 'extra_newline',
-                'raw_data': 'col1,col2,col3\n1,2\n,3\n',
-                'script': "shortname: extra_newline\ntable: extra_newline, http://example.com/extra_newline.txt",
-                'expect_out': 'col1,col2,col3\n1,2,3\n'}
+                 'raw_data': 'col1,col2,col3\n1,2\n,3\n',
+                 'script': "shortname: extra_newline\ntable: extra_newline, http://example.com/extra_newline.txt",
+                 'expect_out': 'col1,col2,col3\n1,2,3\n'}
 
 tests = [simple_csv, autopk_csv, crosstab, autopk_crosstab, skip_csv, extra_newline]
 
 # Create a tuple of all test scripts and expected values
 # (simple_csv, '"a","b","c"\n1,2,3\n4,5,6')
 test_parameters = [(test, test['expect_out']) for test in tests]
+file_location = os.path.dirname(os.path.realpath(__file__))
+retriever_root_dir = os.path.abspath(os.path.join(file_location, os.pardir))
 
 
 def setup_module():
@@ -56,7 +57,7 @@ def setup_module():
             os.makedirs(os.path.join(HOME_DIR, "raw_data", test['name']))
         create_file(test['raw_data'], os.path.join(HOME_DIR, "raw_data", test['name'], test['name'] + '.txt'))
         create_file(test['script'], os.path.join(HOME_DIR, "scripts", test['name'] + '.script'))
-        parse_script_to_json(test['name'], location = os.path.join(HOME_DIR, "scripts"))
+        parse_script_to_json(test['name'], location=os.path.join(HOME_DIR, "scripts"))
         compile_json(os.path.join(HOME_DIR, "scripts", test['name']))
 
 
@@ -64,12 +65,13 @@ def teardown_module():
     """Remove test data and scripts from .retriever directories"""
     for test in tests:
         shutil.rmtree(os.path.join(HOME_DIR, "raw_data", test['name']))
-        os.remove(os.path.join(HOME_DIR , "scripts", test['name'] + '.script'))
-        os.remove(os.path.join(HOME_DIR , "scripts", test['name'] + '.json'))
-        os.remove(os.path.join(HOME_DIR , "scripts", test['name'] + '.py'))
+        os.remove(os.path.join(HOME_DIR, "scripts", test['name'] + '.script'))
+        os.remove(os.path.join(HOME_DIR, "scripts", test['name'] + '.json'))
+        os.remove(os.path.join(HOME_DIR, "scripts", test['name'] + '.py'))
         os.system("rm -r *{}".format(test['name']))
         os.system("rm testdb.sqlite")
 
+
 def get_output_as_csv(dataset, engines, tmpdir, db):
     """Install dataset and return the output as a string version of the csv
 
@@ -88,7 +90,7 @@ def get_output_as_csv(dataset, engines, tmpdir, db):
     if engines.opts["engine"] != 'csv':
         csv_file += '.csv'
     obs_out = file_2string(csv_file)
-    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    os.chdir(retriever_root_dir)
     return obs_out
 
 
@@ -97,16 +99,17 @@ def get_script_module(script_name):
     file, pathname, desc = imp.find_module(script_name, [os.path.join(HOME_DIR, "scripts")])
     return imp.load_module(script_name, file, pathname, desc)
 
+
 mysql_engine, postgres_engine, sqlite_engine, msaccess_engine, csv_engine, download_engine, json_engine, xml_engine = ENGINE_LIST()
 
 
-@pytest.mark.parametrize("dataset, expected",test_parameters)
+@pytest.mark.parametrize("dataset, expected", test_parameters)
 def test_csv_integration(dataset, expected, tmpdir):
     csv_engine.opts = {'engine': 'csv', 'table_name': '{db}_{table}'}
     assert get_output_as_csv(dataset, csv_engine, tmpdir, db=dataset["name"]) == expected
 
 
-@pytest.mark.parametrize("dataset, expected",test_parameters)
+@pytest.mark.parametrize("dataset, expected", test_parameters)
 def test_sqlite_integration(dataset, expected, tmpdir):
     dbfile = os.path.normpath(os.path.join(os.getcwd(), 'testdb.sqlite'))
     sqlite_engine.opts = {'engine': 'sqlite', 'file': dbfile, 'table_name': '{db}_{table}'}
diff --git a/test/test_regression_cli.py b/test/test_regression_cli.py
--- a/test/test_regression_cli.py
+++ b/test/test_regression_cli.py
@@ -9,10 +9,13 @@
     sys.setdefaultencoding('latin-1')
 import pytest
 from retriever.lib.tools import getmd5
-from retriever import HOME_DIR, ENGINE_LIST
+from retriever import ENGINE_LIST
 
 mysql_engine, postgres_engine, sqlite_engine, msaccess_engine, csv_engine, download_engine, json_engine, xml_engine = ENGINE_LIST()
-dir_path = os.path.dirname(os.path.abspath(__file__))
+file_location = os.path.dirname(os.path.realpath(__file__))
+retriever_root_dir = os.path.abspath(os.path.join(file_location, os.pardir))
+working_script_dir = os.path.abspath(os.path.join(retriever_root_dir, "scripts"))
+HOMEDIR = os.path.expanduser('~')
 
 download_md5 = [
     # ('DelMoral2010', '0'),
@@ -28,8 +31,8 @@
 
 
 def get_script_module(script_name):
-    """Load a script module"""
-    file, pathname, desc = imp.find_module(script_name, [os.path.join(HOME_DIR, "scripts")])
+    """Load a script module from the downloaded scripts directory in the retriever"""
+    file, pathname, desc = imp.find_module(script_name, [working_script_dir])
     return imp.load_module(script_name, file, pathname, desc)
 
 
@@ -46,18 +49,17 @@ def get_csv_md5(dataset, engines, tmpdir):
 
 def setup_module():
     """Update retriever scripts and cd to test directory to find data"""
-    os.chdir(os.path.dirname(os.path.abspath(__file__)))
-    os.system("retriever update")
+    os.chdir(retriever_root_dir)
+    os.system('cp -r {0} {1}'.format(os.path.join(retriever_root_dir, "test/raw_data"), retriever_root_dir))
 
 
 def teardown_module():
     """Cleanup temporary output files after testing and return to root directory"""
-    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    os.chdir(retriever_root_dir)
     os.system("rm -r output*")
     os.system("rm -r raw_data/mom2003")
     os.system("rm -r raw_data/EA*")
     os.system("rm testdb.sqlite")
-    os.chdir("..")
 
 
 @pytest.mark.parametrize("dataset, expected", db_md5)
@@ -110,6 +112,7 @@ def test_csv_regression(dataset, expected, tmpdir):
 @pytest.mark.parametrize("dataset, expected", download_md5)
 def test_download_regression(dataset, expected):
     """Check for regression for a particular dataset downloaded only"""
+    os.chdir(retriever_root_dir)
     os.system("retriever download {0} -p raw_data/{0}".format(dataset))
     current_md5 = getmd5(data="raw_data/{0}".format(dataset), data_type='dir', mode="rU")
     assert current_md5 == expected
diff --git a/test/test_retriever.py b/test/test_retriever.py
--- a/test/test_retriever.py
+++ b/test/test_retriever.py
@@ -1,16 +1,14 @@
 """Tests for the Data Retriever"""
 from future import standard_library
-standard_library.install_aliases()
-from imp import reload
-from future.builtins import input
 
+standard_library.install_aliases()
 import os
-import pytest
 import sys
+from imp import reload
+
 reload(sys)
 if hasattr(sys, 'setdefaultencoding'):
     sys.setdefaultencoding('utf-8')
-from io import StringIO
 from retriever.lib.engine import Engine
 from retriever.lib.table import Table
 from retriever.lib.templates import BasicTextTemplate
@@ -32,16 +30,19 @@
                                        shortname='test')
 test_engine.opts = {'database_name': '{db}_abc'}
 HOMEDIR = os.path.expanduser('~')
+file_location = os.path.dirname(os.path.realpath(__file__))
+retriever_root_dir = os.path.abspath(os.path.join(file_location, os.pardir))
 
 
 def setup_module():
-    """"change directory to test directory"""
-    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    """"Make sure you are in the main local retriever directory"""
+    os.chdir(retriever_root_dir)
+    os.system('cp -r {0} {1}'.format(os.path.join(retriever_root_dir, "test/raw_data"), retriever_root_dir))
 
 
 def teardown_method():
-    """Cleanup temporary output files after testing"""
-    os.chdir("..")
+    """Make sure you are in the main local retriever directory after these tests"""
+    os.chdir(retriever_root_dir)
 
 
 def test_auto_get_columns():
@@ -141,10 +142,9 @@ def test_find_file_absent():
 def test_find_file_present():
     """Test if existing datafile is found
 
-    Using the AvianBodySize dataset which is included for regression testing
-    Because all testing code and data is located in ./test/ it is necessary to
-    move into this directory for DATA_SEARCH_PATHS to work properly.
-
+    Using the AvianBodySize dataset which is included for regression testing.
+    We copy the raw_data directory to retriever_root_dir which is the current working directory.
+    This enables the data to be in the DATA_SEARCH_PATHS.
     """
     test_engine.script.shortname = 'AvianBodySize'
     assert test_engine.find_file('avian_ssd_jan07.txt') == os.path.normpath(
@@ -195,7 +195,7 @@ def test_getmd5_lines():
 def test_getmd5_path():
     """Test md5 sum calculation given a path to data source"""
     data_file = create_file('a,b,c\n1,2,3\n4,5,6\n')
-    assert getmd5(data=data_file, data_type='file',mode='rU') == '0bec5bf6f93c547bc9c6774acaf85e1a'
+    assert getmd5(data=data_file, data_type='file', mode='rU') == '0bec5bf6f93c547bc9c6774acaf85e1a'
 
 
 def test_json2csv():
@@ -246,101 +246,117 @@ def test_sort_csv():
 
 def test_is_empty_null_string():
     """Test for null string"""
-    assert(is_empty("") == True)
+    assert is_empty("") == True
 
 
 def test_is_empty_empty_list():
     """Test for empty list"""
-    assert(is_empty([]) == True)
+    assert is_empty([]) == True
 
 
 def test_is_empty_not_null_string():
     """Test for non-null string"""
-    assert(is_empty("not empty") == False)
+    assert is_empty("not empty") == False
 
 
 def test_is_empty_not_empty_list():
     """Test for not empty list"""
-    assert(is_empty(["not empty"]) == False)
+    assert is_empty(["not empty"]) == False
 
 
 def test_clean_input_empty_input_ignore_empty(monkeypatch):
     """Test with empty input ignored"""
+
     def mock_input(prompt):
         return ""
+
     monkeypatch.setattr('retriever.lib.datapackage.input', mock_input)
-    assert(clean_input("", ignore_empty=True) == "")
+    assert clean_input("", ignore_empty=True) == ""
 
 
 def test_clean_input_empty_input_not_ignore_empty(monkeypatch):
     """Test with empty input not ignored"""
+
     def mock_input(prompt):
         mock_input.counter += 1
         if mock_input.counter <= 1:
             return ""
         else:
             return "not empty"
+
     mock_input.counter = 0
     monkeypatch.setattr('retriever.lib.datapackage.input', mock_input)
-    assert(clean_input("", ignore_empty=False) == "not empty")
+    assert clean_input("", ignore_empty=False) == "not empty"
 
 
 def test_clean_input_string_input(monkeypatch):
     """Test with non-empty input"""
+
     def mock_input(prompt):
         return "not empty"
+
     monkeypatch.setattr('retriever.lib.datapackage.input', mock_input)
-    assert(clean_input("") == "not empty")
+    assert clean_input("") == "not empty"
 
 
 def test_clean_input_empty_list_ignore_empty(monkeypatch):
     """Test with empty list ignored"""
+
     def mock_input(prompt):
         return ",  ,   ,"
+
     monkeypatch.setattr('retriever.lib.datapackage.input', mock_input)
-    assert(clean_input("", ignore_empty=True, split_char=",") == [])
+    assert clean_input("", ignore_empty=True, split_char=",") == []
 
 
 def test_clean_input_empty_list_not_ignore_empty(monkeypatch):
     """Test with empty list not ignored"""
+
     def mock_input(prompt):
         mock_input.counter += 1
         if mock_input.counter <= 1:
             return ",  ,   ,"
         else:
             return "1  ,    2,  3,"
+
     mock_input.counter = 0
     monkeypatch.setattr('retriever.lib.datapackage.input', mock_input)
-    assert(clean_input("", split_char=",") == ["1", "2", "3"])
+    assert clean_input("", split_char=",") == ["1", "2", "3"]
 
 
 def test_clean_input_not_empty_list(monkeypatch):
     """Test with list input"""
+
     def mock_input(prompt):
         return "1,    2,     3"
+
     monkeypatch.setattr('retriever.lib.datapackage.input', mock_input)
-    assert(clean_input("", ignore_empty=True, split_char=',', dtype=None) == ["1", "2", "3"])
+    assert clean_input("", ignore_empty=True, split_char=',', dtype=None) == ["1", "2", "3"]
 
 
 def test_clean_input_bool(monkeypatch):
     """Test with correct datatype input"""
+
     def mock_input(prompt):
         return "True "
+
     monkeypatch.setattr('retriever.lib.datapackage.input', mock_input)
-    assert(clean_input("", dtype=bool) == "True")
+    assert clean_input("", dtype=bool) == "True"
 
 
 def test_clean_input_not_bool(monkeypatch):
     """Test with incorrect datatype input"""
+
     def mock_input(prompt):
         mock_input.counter += 1
         if mock_input.counter <= 1:
             return "non bool input"
         else:
             return "True "
+
     mock_input.counter = 0
     monkeypatch.setattr('retriever.lib.datapackage.input', mock_input)
-    assert(clean_input("", dtype=bool) == "True")
+    assert clean_input("", dtype=bool) == "True"
 
 
 def test_add_dialect():