Repository: wireservice/csvkit
Issue: Unicode tests fail with Python 2

I have trouble getting tests right from the repository freshly cloned from master.

Some "unicode tests" won't pass with Python 2 (UnicodeDecodeError). Those are _test_ndjson_ for csvjson, _test_unicode_ for csvlook, _test_sort_date_ for csvsort, and  _test_sort_string_reverse_ for csvsort.

Sample output:

```
ERROR: test_ndjson (tests.test_utilities.test_csvjson.TestCSVJSON)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/quentin/work/csvkit/tests/test_utilities/test_csvjson.py", line 108, in test_ndjson
    '{"text": "Unicode! Σ", "float": null, "datetime": null, "boolean": null, "time": null, "date": null, "integer": null}',
  File "/home/quentin/work/csvkit/tests/utils.py", line 56, in assertLines
    lines = self.get_output_as_list(args)
  File "/home/quentin/work/csvkit/tests/utils.py", line 44, in get_output_as_list
    return self.get_output(args).split('\\n')
  File "/home/quentin/work/csvkit/tests/utils.py", line 37, in get_output
    utility.main()
  File "/home/quentin/work/csvkit/csvkit/utilities/csvjson.py", line 168, in main
    table = agate.Table.from_csv(self.input_file, sniff_limit=self.args.sniff_limit, column_types=self.get_column_types(), **self.reader_kwargs)
  File "/home/quentin/work/csvkit/.tox/pypy/site-packages/agate/table/__init__.py", line 362, in from_csv
    contents = contents.encode('utf-8')
UnicodeDecodeError: 'ascii' codec can't decode byte 0xce in position 295: ordinal not in range(128)
```

Is it something on my side only?


Changed files:
csvkit/utilities/csvlook.py, csvkit/utilities/csvjson.py, csvkit/utilities/csvsort.py, csvkit/cli.py

diff --git a/csvkit/cli.py b/csvkit/cli.py
index 37f8c04..196f87e 100644
--- a/csvkit/cli.py
+++ b/csvkit/cli.py
@@ -195,7 +195,7 @@ class CSVKitUtility(object):
         else:
             (_, extension) = os.path.splitext(path)
 
-            if extension == u'.gz':
+            if extension == '.gz':
                 f = LazyFile(gzip.open, path, mode, **kwargs)
             elif extension == '.bz2':
                 if six.PY2:
@@ -207,6 +207,22 @@ class CSVKitUtility(object):
 
         return f
 
+    def file_or_path(self):
+        """
+        Returns the input path unless the input is stdin or an archive file.
+        """
+        path = self.args.input_path
+
+        if not path or path == '-':
+            return self.input_file
+        else:
+            (_, extension) = os.path.splitext(path)
+
+            if extension in ('.gz', '.bz2'):
+                return self.input_file
+            else:
+                return path
+
     def _extract_csv_reader_kwargs(self):
         """
         Extracts those from the command-line arguments those would should be passed through to the input CSV reader(s).
diff --git a/csvkit/utilities/csvjson.py b/csvkit/utilities/csvjson.py
index f84e640..9eff839 100644
--- a/csvkit/utilities/csvjson.py
+++ b/csvkit/utilities/csvjson.py
@@ -166,7 +166,7 @@ class CSVJSON(CSVKitUtility):
                         data[column] = None
                 dump_json(data, newline=True)
         else:
-            table = agate.Table.from_csv(self.input_file, sniff_limit=self.args.sniff_limit, column_types=self.get_column_types(), **self.reader_kwargs)
+            table = agate.Table.from_csv(self.file_or_path(), sniff_limit=self.args.sniff_limit, column_types=self.get_column_types(), **self.reader_kwargs)
             table.to_json(self.output_file, key=self.args.key, newline=self.args.streamOutput, **json_kwargs)
 
 
diff --git a/csvkit/utilities/csvlook.py b/csvkit/utilities/csvlook.py
index 3e22da4..84aa8a5 100644
--- a/csvkit/utilities/csvlook.py
+++ b/csvkit/utilities/csvlook.py
@@ -22,7 +22,7 @@ class CSVLook(CSVKitUtility):
                                     help='Disable type inference when parsing the input.')
 
     def main(self):
-        table = agate.Table.from_csv(self.input_file, sniff_limit=self.args.sniff_limit, header=not self.args.no_header_row, column_types=self.get_column_types(), line_numbers=self.args.line_numbers, **self.reader_kwargs)
+        table = agate.Table.from_csv(self.file_or_path(), sniff_limit=self.args.sniff_limit, header=not self.args.no_header_row, column_types=self.get_column_types(), line_numbers=self.args.line_numbers, **self.reader_kwargs)
         table.print_table(output=self.output_file, max_rows=self.args.max_rows, max_columns=self.args.max_columns, max_column_width=self.args.max_column_width)
 
 
diff --git a/csvkit/utilities/csvsort.py b/csvkit/utilities/csvsort.py
index b3af78f..2fce2fc 100644
--- a/csvkit/utilities/csvsort.py
+++ b/csvkit/utilities/csvsort.py
@@ -26,7 +26,7 @@ class CSVSort(CSVKitUtility):
             self.print_column_names()
             return
 
-        table = agate.Table.from_csv(self.input_file, sniff_limit=self.args.sniff_limit, header=not self.args.no_header_row, column_types=self.get_column_types(), **self.reader_kwargs)
+        table = agate.Table.from_csv(self.file_or_path(), sniff_limit=self.args.sniff_limit, header=not self.args.no_header_row, column_types=self.get_column_types(), **self.reader_kwargs)
         column_ids = parse_column_identifiers(self.args.columns, table.column_names, column_offset=self.get_column_offset())
         table = table.order_by(column_ids, reverse=self.args.reverse)
         table.to_csv(self.output_file, **self.writer_kwargs)