Repository: weecology/retriever

Issue:
Add integration testing
We should be able to add small scale integration testing by adding small data files for testing to `test/raw_data` and scripts for those data files to `test/scripts`.



Discussion:
I have a development version of this in my fork:
https://github.com/ethanwhite/retriever/tree/integration-testing


Resolving patch:
diff --git a/engines/sqlite.py b/engines/sqlite.py
--- a/engines/sqlite.py
+++ b/engines/sqlite.py
@@ -74,8 +74,10 @@ def insert_data_from_file(self, filename):
             filename = os.path.abspath(filename)
             try:
                 bulk_insert_statement = self.get_bulk_insert_statement()
+                line_endings = set(['\n', '\r', '\r\n'])
                 with open(filename, 'r') as data_file:
                     data_chunk = data_file.readlines(CHUNK_SIZE)
+                    data_chunk = [line.rstrip('\r\n') for line in data_chunk if line not in line_endings]
                     del(data_chunk[:self.table.header_rows])
                     while data_chunk:
                         data_chunk_split = [row.split(self.table.delimiter)
diff --git a/lib/engine.py b/lib/engine.py
--- a/lib/engine.py
+++ b/lib/engine.py
@@ -74,6 +74,9 @@ def add_to_table(self, data_source):
             for line in lines:
                 split_line = self.table.split_on_delimiter(line)
                 initial_cols = len(self.table.columns) - (3 if hasattr(self.table, "ct_names") else 2)
+                # add one if auto increment is not set to get the right initial columns
+                if not self.table.columns[0][1][0] == "pk-auto":
+                    initial_cols += 1
                 begin = split_line[:initial_cols]
                 rest = split_line[initial_cols:]
                 n = 0
@@ -408,7 +411,6 @@ def download_file(self, url, filename):
             print("Downloading " + filename + "...")
             urlretrieve(url, path)
 
-
     def download_files_from_archive(self, url, filenames, filetype="zip",
                                     keep_in_dir=False, archivename=None):
         """Downloads files from an archive into the raw data directory.
diff --git a/lib/tools.py b/lib/tools.py
--- a/lib/tools.py
+++ b/lib/tools.py
@@ -361,7 +361,7 @@ def sort_csv(filename):
     os.remove(os.path.normpath("tempfile"))
     return filename
 
-#     output_file = os.path.normpath(os.path.join(test_dir, output))
+
 def create_file(data, output='output_file'):
     """Writes a string to a file for use by tests"""
     output_file = os.path.normpath(output)


Test patch:
diff --git a/test/test_integration.py b/test/test_integration.py
--- a/test/test_integration.py
+++ b/test/test_integration.py
@@ -3,28 +3,40 @@
 import imp
 import os
 import shutil
+import pytest
 
 from retriever.lib.compile import compile_script
 from retriever import HOME_DIR, ENGINE_LIST
 from retriever.lib.tools import file_2string
+from retriever.lib.tools import create_file
 
 
 simple_csv = {'name': 'simple_csv',
               'raw_data': "a,b,c\n1,2,3\n4,5,6\n",
               'script': "shortname: simple_csv\ntable: simple_csv, http://example.com/simple_csv.txt",
-              'expect_out': '"a","b","c"\n1,2,3\n4,5,6'}
+              'expect_out': 'a,b,c\n1,2,3\n4,5,6\n'}
+
+autopk_csv = {'name': 'autopk_csv',
+              'raw_data': "a,b,c\n1,2,3\n4,5,6\n",
+              'script': "shortname: autopk_csv\ntable: autopk_csv, http://example.com/autopk_csv.txt\n*column: record_id, pk-auto\n*column: a, int\n*column: b, int\n*column: c, int",
+              'expect_out': 'record_id,a,b,c\n1,1,2,3\n2,4,5,6\n'}
 
 crosstab = {'name': 'crosstab',
-            'raw_data': "a,b,c1,c2\n1,1,1.1,1.2\n1,2,2.1,2.2",
-            'script': "shortname: crosstab\ntable: crosstab, http://example.com/crosstab.txt\n*column: record_id, pk-auto\n*column: a, int\n*column: b, int\n*ct_column: c\n*column: val, ct-double\n*ct_names: c1,c2",
-            'expect_out': '"record_id","a","b","c","val"\n1,1,1,"c1",1.1\n2,1,1,"c2",1.2\n3,1,2,"c1",2.1\n4,1,2,"c2",2.2'}
+            'raw_data': "a,b,c1,c2\n1,1,1.1,1.2\n1,2,2.1,2.2\n",
+            'script': "shortname: crosstab\ntable: crosstab, http://example.com/crosstab.txt\n*column: a, int\n*column: b, int\n*ct_column: c\n*column: val, ct-double\n*ct_names: c1,c2",
+            'expect_out': 'a,b,c,val\n1,1,c1,1.1\n1,1,c2,1.2\n1,2,c1,2.1\n1,2,c2,2.2\n'}
+
+autopk_crosstab = {'name': 'autopk_crosstab',
+            'raw_data': "a,b,c1,c2\n1,1,1.1,1.2\n1,2,2.1,2.2\n",
+            'script': "shortname: autopk_crosstab\ntable: autopk_crosstab, http://example.com/autopk_crosstab.txt\n*column: record_id, pk-auto\n*column: a, int\n*column: b, int\n*ct_column: c\n*column: val, ct-double\n*ct_names: c1,c2",
+            'expect_out': 'record_id,a,b,c,val\n1,1,1,c1,1.1\n2,1,1,c2,1.2\n3,1,2,c1,2.1\n4,1,2,c2,2.2\n'}
+
 
-extra_newline = {'name': 'extra_newline',
-              'raw_data': 'col1,col2,col3\n1,2\n,3\n',
-              'script': "shortname: extra_newline\ntable: extra_newline, http://example.com/extra_newline.txt",
-              'expect_out': '"col1","col2","col3"\n1,2,3'}
+tests = [simple_csv, autopk_csv, crosstab, autopk_crosstab]
 
-tests = [simple_csv, crosstab, extra_newline]
+# Create a tuple of all test scripts and expected values
+# (simple_csv, '"a","b","c"\n1,2,3\n4,5,6')
+test_parameters = [(test, test['expect_out']) for test in tests]
 
 
 def setup_module():
@@ -32,10 +44,8 @@ def setup_module():
     for test in tests:
         if not os.path.exists(os.path.join(HOME_DIR, "raw_data", test['name'])):
             os.makedirs(os.path.join(HOME_DIR, "raw_data", test['name']))
-        with open(os.path.join(HOME_DIR, "raw_data", test['name'], test['name'] + '.txt'), 'w') as data_file:
-            data_file.write(test['raw_data'])
-        with open(os.path.join(HOME_DIR, "scripts", test['name'] + '.script'), 'w') as script_file:
-            script_file.write(test['script'])
+        create_file(test['raw_data'], os.path.join(HOME_DIR, "raw_data", test['name'], test['name'] + '.txt'))
+        create_file(test['script'], os.path.join(HOME_DIR, "scripts", test['name'] + '.script'))
         compile_script(os.path.join(HOME_DIR, "scripts", test['name']))
 
 
@@ -44,7 +54,29 @@ def teardown_module():
     for test in tests:
         shutil.rmtree(os.path.join(HOME_DIR, "raw_data", test['name']))
         os.remove(os.path.join(HOME_DIR, "scripts", test['name'] + '.script'))
-        os.remove(test['name']+'_'+test['name']+'.txt')
+        os.system("rm -r *{}".format(test['name']))
+        os.system("rm testdb.sqlite")
+
+def get_output_as_csv(dataset, engines, tmpdir, db):
+    """Install dataset and return the output as a string version of the csv
+
+    The string version of the csv output returned by this function can be compared
+    directly to the expect_out values in the dataset test dictionaries.
+    """
+    workdir = tmpdir.mkdtemp()
+    workdir.chdir()
+    script_module = get_script_module(dataset["name"])
+    script_module.SCRIPT.download(engines)
+    script_module.SCRIPT.engine.final_cleanup()
+    script_module.SCRIPT.engine.to_csv()
+    # get filename and append .csv
+    csv_file = engines.opts['table_name'].format(db=db, table=dataset["name"])
+    # csv engine already has the .csv extension
+    if engines.opts["engine"] != 'csv':
+        csv_file += '.csv'
+    obs_out = file_2string(csv_file)
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    return obs_out
 
 
 def get_script_module(script_name):
@@ -53,27 +85,49 @@ def get_script_module(script_name):
     return imp.load_module(script_name, file, pathname, desc)
 
 mysql_engine, postgres_engine, sqlite_engine, msaccess_engine, csv_engine, download_engine, json_engine, xml_engine = ENGINE_LIST()
-csv_engine.opts = {'engine': 'csv', 'table_name': './{db}_{table}.txt'}
-
-
-def test_csv_from_csv():
-    simple_csv_module = get_script_module('simple_csv')
-    simple_csv_module.SCRIPT.download(csv_engine)
-    simple_csv_module.SCRIPT.engine.disconnect()
-    obs_out = file_2string("simple_csv_simple_csv.txt")
-    assert obs_out == simple_csv['expect_out']
-
-
-def test_crosstab_from_csv():
-    crosstab_module = get_script_module('crosstab')
-    crosstab_module.SCRIPT.download(csv_engine)
-    crosstab_module.SCRIPT.engine.disconnect()
-    obs_out = file_2string("crosstab_crosstab.txt")
-    assert obs_out == crosstab['expect_out']
-
-def test_extra_newline():
-    extra_newline_module = get_script_module('extra_newline')
-    extra_newline_module.SCRIPT.download(csv_engine)
-    extra_newline_module.SCRIPT.engine.disconnect()
-    obs_out = file_2string("extra_newline_extra_newline.txt")
-    assert obs_out == extra_newline['expect_out']
+
+
+@pytest.mark.parametrize("dataset, expected",test_parameters)
+def test_csv_integration(dataset, expected, tmpdir):
+    csv_engine.opts = {'engine': 'csv', 'table_name': '{db}_{table}'}
+    assert get_output_as_csv(dataset, csv_engine, tmpdir, db=dataset["name"]) == expected
+
+
+@pytest.mark.parametrize("dataset, expected",test_parameters)
+def test_sqlite_integration(dataset, expected, tmpdir):
+    dbfile = os.path.normpath(os.path.join(os.getcwd(), 'testdb.sqlite'))
+    sqlite_engine.opts = {'engine': 'sqlite', 'file': dbfile, 'table_name': '{db}_{table}'}
+    os.system("rm testdb.sqlite")
+    assert get_output_as_csv(dataset, sqlite_engine, tmpdir, dataset["name"]) == expected
+
+
+@pytest.mark.parametrize("dataset, expected", test_parameters)
+def test_xmlengine_integration(dataset, expected, tmpdir):
+    """Check for xmlenginee regression"""
+    xml_engine.opts = {'engine': 'xml', 'table_name': '{db}_{table}'}
+    assert get_output_as_csv(dataset, xml_engine, tmpdir, db=dataset["name"]) == expected
+
+
+@pytest.mark.parametrize("dataset, expected", test_parameters)
+def test_jsonengine_integration(dataset, expected, tmpdir):
+    """Check for jsonenginee regression"""
+    json_engine.opts = {'engine': 'json', 'table_name': '{db}_{table}'}
+    assert get_output_as_csv(dataset, json_engine, tmpdir, db=dataset["name"]) == expected
+
+
+@pytest.mark.parametrize("dataset, expected", test_parameters)
+def test_postgres_integration(dataset, expected, tmpdir):
+    """Check for postgres regression"""
+    os.system('psql -U postgres -d testdb -h localhost -c "DROP SCHEMA IF EXISTS testschema CASCADE"')
+    postgres_engine.opts = {'engine': 'postgres', 'user': 'postgres', 'password': "", 'host': 'localhost', 'port': 5432,
+                            'database': 'testdb', 'database_name': 'testschema', 'table_name': '{db}.{table}'}
+    assert get_output_as_csv(dataset, postgres_engine, tmpdir, db=postgres_engine.opts['database_name']) == expected
+
+
+@pytest.mark.parametrize("dataset, expected", test_parameters)
+def test_mysql_integration(dataset, expected, tmpdir):
+    """Check for mysql regression"""
+    os.system('mysql -u travis -Bse "DROP DATABASE IF EXISTS testdb"')
+    mysql_engine.opts = {'engine': 'mysql', 'user': 'travis', 'password': '', 'host': 'localhost', 'port': 3306,
+                         'database_name': 'testdb', 'table_name': '{db}.{table}'}
+    assert get_output_as_csv(dataset, mysql_engine, tmpdir, db=mysql_engine.opts['database_name']) == expected
diff --git a/test/test_regression_cli.py b/test/test_regression_cli.py
--- a/test/test_regression_cli.py
+++ b/test/test_regression_cli.py
@@ -46,12 +46,13 @@ def get_csv_md5(dataset, engines, tmpdir):
 
 def setup_module():
     """Update retriever scripts and cd to test directory to find data"""
-    os.chdir("./test/")
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
     os.system("retriever update")
 
 
 def teardown_module():
     """Cleanup temporary output files after testing and return to root directory"""
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
     os.system("rm -r output*")
     os.system("rm -r raw_data/mom2003")
     os.system("rm -r raw_data/EA*")
@@ -86,14 +87,14 @@ def test_mysql_regression(dataset, expected, tmpdir):
 
 
 @pytest.mark.parametrize("dataset, expected", db_md5)
-def test_xmlenginee_regression(dataset, expected, tmpdir):
+def test_xmlengine_regression(dataset, expected, tmpdir):
     """Check for xmlenginee regression"""
     xml_engine.opts = {'engine': 'xml', 'table_name': 'output_file_{table}.xml'}
     assert get_csv_md5(dataset, xml_engine, tmpdir) == expected
 
 
 @pytest.mark.parametrize("dataset, expected", db_md5)
-def test_jsonenginee_regression(dataset, expected, tmpdir):
+def test_jsonengine_regression(dataset, expected, tmpdir):
     """Check for jsonenginee regression"""
     json_engine.opts = {'engine': 'json', 'table_name': 'output_file_{table}.json'}
     assert get_csv_md5(dataset, json_engine, tmpdir) == expected
diff --git a/test/test_retriever.py b/test/test_retriever.py
--- a/test/test_retriever.py
+++ b/test/test_retriever.py
@@ -30,13 +30,11 @@
                                        shortname='test')
 test_engine.opts = {'database_name': '{db}_abc'}
 HOMEDIR = os.path.expanduser('~')
-test_dir = os.path.dirname(os.path.abspath(__file__))
 
 
 def setup_module():
     """"change directory to test directory"""
-    dir_path = os.path.dirname(os.path.abspath(__file__))
-    os.chdir(dir_path)
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 
 def teardown_method():
@@ -141,8 +139,6 @@ def test_find_file_present():
     test_engine.script.shortname = 'AvianBodySize'
     assert test_engine.find_file('avian_ssd_jan07.txt') == os.path.normpath(
         'raw_data/AvianBodySize/avian_ssd_jan07.txt')
-    os.system("rm -r raw_data/avianbodysize")
-    os.chdir('..')
 
 
 def test_format_data_dir():
@@ -196,7 +192,7 @@ def test_json2csv():
     """Test json2csv function
     creates a json file and tests the md5 sum calculation"""
     json_file = create_file("""[ {"User": "Alex", "Country": "US", "Age": "25"} ]""", 'output.json')
-    output_json = json2csv(json_file, "test/output_json.csv", header_values=["User", "Country", "Age"])
+    output_json = json2csv(json_file, "output_json.csv", header_values=["User", "Country", "Age"])
     obs_out = file_2string(output_json)
     os.remove(output_json)
     assert obs_out == 'User,Country,Age\nAlex,US,25'
@@ -214,7 +210,7 @@ def test_xml2csv():
                            "<Country>US</Country>S\n"
                            "<Age>24</Age>\n"
                            "</row>\n</root>", 'output.xml')
-    output_xml = xml2csv(xml_file, "test/output_xml.csv", header_values=["User", "Country", "Age"])
+    output_xml = xml2csv(xml_file, "output_xml.csv", header_values=["User", "Country", "Age"])
     obs_out = file_2string(output_xml)
     os.remove(output_xml)
     assert obs_out == "User,Country,Age\nAlex,US,25\nAlex,PT,25\nBen,US,24"