Repository: weecology/retriever

Issue:
Error on custom table name.
The current state of the retriever uses the required_opts  `"table_name"` to customize the installation of dataset into dataset. This works fine for single table datasets however, it over rides the tables when the dataset contains multi tables. The best way to use this option is by using this custom `table_name` as a prefix to the table name. 

For example:

"{db}.{prefix_name}_{table}") 



Discussion:
henrykironde: I feel like this is a hacky way to solve this issue, but it is one of those edge cases.

As mention in the issue #1117, if users follow the format of the required opts for the table-name provided in the engine, we do not get this issue.

_For example_ 
**Mysql and Postgres:** `{db}.{table}`, `{db}.prefix{table}`
**Sqlite:** `{db}_{table}`, `{db}_prefix{table}`

When a user provides something like `retriever install csv portal -t march`, the engine will fail to maintain the format for the table name.

To avoid this, we are getting the default required opt for that engine `{db}_{table}.csv` and replacing the `{table}` value with `prefix{table}`.
The final format will look like `{db}_prefix{table}.csv`
henrykironde: I will add some multi table tests as I add the fetch function. #1069

Patch:
diff --git a/retriever/engines/__init__.py b/retriever/engines/__init__.py
index 0f4de6b..4156f7b 100644
--- a/retriever/engines/__init__.py
+++ b/retriever/engines/__init__.py
@@ -1,4 +1,6 @@
 """Contains DBMS-specific Engine implementations."""
+import os
+
 from retriever.lib.engine import Engine
 
 engines = [
@@ -47,6 +49,11 @@ def choose_engine(opts, choice=True):
             if (enginename == thisengine.name.lower() or thisengine.abbreviation and
                     enginename == thisengine.abbreviation):
                 engine = thisengine
+    if 'table_name' in opts:
+        if opts['table_name'] and "{table}" not in opts['table_name'] or "{db}" not in opts['table_name']:
+            for opt in engine.required_opts:
+                if opt[0] == 'table_name':
+                    raise Exception('Accepted Table format {fom}'.format(fom=opt[2]))
 
     engine.opts = opts
     return engine
diff --git a/retriever/engines/csvengine.py b/retriever/engines/csvengine.py
index 18e3bf2..871de5e 100644
--- a/retriever/engines/csvengine.py
+++ b/retriever/engines/csvengine.py
@@ -88,8 +88,7 @@ class engine(Engine):
     def to_csv(self):
         """Export sorted version of CSV file"""
         for keys in self.script.tables:
-            table_name = self.opts['table_name'].format(db=self.db_name, table=keys)
-            sort_csv(table_name)
+            sort_csv(self.table_name())
 
     def get_connection(self):
         """Gets the db connection."""
diff --git a/retriever/engines/jsonengine.py b/retriever/engines/jsonengine.py
index 4b2a108..aee229d 100644
--- a/retriever/engines/jsonengine.py
+++ b/retriever/engines/jsonengine.py
@@ -114,7 +114,7 @@ class engine(Engine):
     def to_csv(self):
         """Export table from json engine to CSV file"""
         for keys in list(self.script.tables):
-            table_name = self.opts['table_name'].format(db=self.db_name, table=keys)
+            table_name = self.table_name()
             header = self.script.tables[keys].get_insert_columns(join=False, create=True)
             csv_outfile = json2csv(table_name, header_values=header)
             sort_csv(csv_outfile)
diff --git a/retriever/engines/xmlengine.py b/retriever/engines/xmlengine.py
index e03e43c..cc5816f 100644
--- a/retriever/engines/xmlengine.py
+++ b/retriever/engines/xmlengine.py
@@ -109,7 +109,7 @@ class engine(Engine):
     def to_csv(self):
         """Export table from xml engine to CSV file."""
         for keys in list(self.script.tables):
-            table_name = self.opts['table_name'].format(db=self.db_name, table=keys)
+            table_name = self.table_name()
             header = self.script.tables[keys].get_insert_columns(join=False, create=True)
             csv_outfile = xml2csv(table_name, header_values=header)
             sort_csv(csv_outfile)


Test patch:
diff --git a/test/test_regression.py b/test/test_regression.py
index a551828..15081ec 100644
--- a/test/test_regression.py
+++ b/test/test_regression.py
@@ -64,7 +64,7 @@ def setup_module():
 def teardown_module():
     """Cleanup temporary output files and return to root directory."""
     os.chdir(retriever_root_dir)
-    os.system("rm -r output*")
+    os.system("rm -r *output*")
     shutil.rmtree(os.path.join(retriever_root_dir, "raw_data"))
     os.system("rm testdb.sqlite")
 
@@ -141,8 +141,8 @@ def test_xmlengine_regression(dataset, expected, tmpdir):
     """Check for xmlenginee regression."""
     xml_engine.opts = {
         'engine': 'xml',
-        'table_name': 'output_file_{table}.xml'}
-    interface_opts = {'table_name': 'output_file_{table}.xml'}
+        'table_name': '{db}_output_{table}.xml'}
+    interface_opts = {'table_name': '{db}_output_{table}.xml'}
     assert get_csv_md5(dataset, xml_engine, tmpdir, install_xml, interface_opts) == expected
 
 
@@ -151,8 +151,8 @@ def test_jsonengine_regression(dataset, expected, tmpdir):
     """Check for jsonenginee regression."""
     json_engine.opts = {
         'engine': 'json',
-        'table_name': 'output_file_{table}.json'}
-    interface_opts = {'table_name': 'output_file_{table}.json'}
+        'table_name': '{db}_output_{table}.json'}
+    interface_opts = {'table_name': '{db}_output_{table}.json'}
     assert get_csv_md5(dataset, json_engine, tmpdir, install_json, interface_opts) == expected
 
 
@@ -161,8 +161,8 @@ def test_csv_regression(dataset, expected, tmpdir):
     """Check csv regression."""
     csv_engine.opts = {
         'engine': 'csv',
-        'table_name': 'output_file_{table}.csv'}
-    interface_opts = {'table_name': 'output_file_{table}.csv'}
+        'table_name': '{db}_output_{table}.csv'}
+    interface_opts = {'table_name': '{db}_output_{table}.csv'}
     assert get_csv_md5(dataset, csv_engine, tmpdir, install_csv, interface_opts) == expected