Repository: wireservice/csvkit

Issue:
csvstat doen't give counts for some columns
csvstat is great, and usually gives a sense of the mode of the data, via a count of how many rows have the most frequent values:
<pre>
 4. LAST_NAME
        <type 'unicode'>
        Nulls: True
        Unique values: 307123
        5 most frequent values:
                SMITH:  28155
                JOHNSON:        23713
                MARTINEZ:       18465
                MILLER: 16916
                BROWN:  15428
        Max length: 28
</pre>
But sometimes it doesn't, perhaps when there less than some predefined number of unique values:
 
<pre>
28. STATUS
        <type 'unicode'>
        Nulls: False
        Values: Active, Inactive
</pre>

I'd like to get counts of the top 5 most frequent values for all columns, as long as there are any repeated values.


Comments:

Patch:
diff --git a/CHANGELOG b/CHANGELOG
index a9645ce..fb00cfa 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -11,6 +11,7 @@ Backwards-incompatible changes:
 
 * The --doublequote long flag is gone, and the -b short flag is now an alias for --no-doublequote.
 * When using the --columns or --not-columns options, you must not have spaces around the comma-separated values, unless the column names contain spaces.
+* When sorting, null values are now greater than other values instead of less than.
 * CSVKitReader, CSVKitWriter, CSVKitDictReader, and CSVKitDictWriter have been removed. Use agate.csv.reader, agate.csv.writer, agate.csv.DictReader and agate.csv.DictWriter.
 * Drop Python 2.6 support.
 
@@ -39,8 +40,10 @@ Fixes:
 * csvclean with standard input works on Windows.
 * csvgrep returns the input file's line numbers if the --linenumbers flag is set.
 * csvgrep can match multiline values.
+* csvgrep correctly operates on ragged rows.
 * csvsql correctly escapes `%` characters in SQL queries.
 * csvstack supports stacking a single file.
+* csvstat always reports frequencies.
 * FilteringCSVReader's any_match argument works correctly.
 * All tools handle empty files without error.
 
diff --git a/csvkit/grep.py b/csvkit/grep.py
index 975d111..58fc0ee 100644
--- a/csvkit/grep.py
+++ b/csvkit/grep.py
@@ -64,7 +64,11 @@ class FilteringCSVReader(six.Iterator):
 
     def test_row(self, row):
         for idx, test in self.patterns.items():
-            result = test(row[idx])
+            try:
+                value = row[idx]
+            except IndexError:
+                value = ''
+            result = test(value)
             if self.any_match:
                 if result:
                     return not self.inverse  # True
diff --git a/csvkit/utilities/csvsort.py b/csvkit/utilities/csvsort.py
index 4d043b8..afe439a 100644
--- a/csvkit/utilities/csvsort.py
+++ b/csvkit/utilities/csvsort.py
@@ -35,7 +35,7 @@ class CSVSort(CSVKitUtility):
 
         table = agate.Table.from_csv(self.input_file, sniff_limit=self.args.sniff_limit, header=not self.args.no_header_row, column_types=self.get_column_types(), **self.reader_kwargs)
         column_ids = parse_column_identifiers(self.args.columns, table.column_names, column_offset=self.get_column_offset())
-        table = table.order_by(lambda row: [(row[column_id] is not None, row[column_id]) for column_id in column_ids], reverse=self.args.reverse)
+        table = table.order_by(column_ids, reverse=self.args.reverse)
         table.to_csv(self.output_file, **self.writer_kwargs)
 
 
diff --git a/csvkit/utilities/csvstat.py b/csvkit/utilities/csvstat.py
index e6b143c..56e8019 100644
--- a/csvkit/utilities/csvstat.py
+++ b/csvkit/utilities/csvstat.py
@@ -142,14 +142,13 @@ class CSVStat(CSVKitUtility):
 
                     self.output_file.write('\tUnique values: %i\n' % len(stats['unique']))
 
-                    if len(stats['unique']) != len(values):
-                        self.output_file.write('\t%i most frequent values:\n' % MAX_FREQ)
-                        for value, count in stats['freq']:
-                            self.output_file.write(('\t\t%s:\t%s\n' % (six.text_type(value), count)))
-
                     if c.type == six.text_type:
                         self.output_file.write('\tMax length: %i\n' % stats['len'])
 
+                self.output_file.write('\t%i most frequent values:\n' % MAX_FREQ)
+                for value, count in stats['freq']:
+                    self.output_file.write(('\t\t%s:\t%s\n' % (six.text_type(value), count)))
+
         if not operations:
             self.output_file.write('\n')
             self.output_file.write('Row count: %s\n' % tab.count_rows())


Test patch:
diff --git a/tests/test_grep.py b/tests/test_grep.py
index 4a293f0..ade3c9b 100644
--- a/tests/test_grep.py
+++ b/tests/test_grep.py
@@ -102,6 +102,16 @@ class TestGrep(unittest.TestCase):
         except ColumnIdentifierError:
             pass
 
+    def test_index_out_of_range(self):
+        fcr = FilteringCSVReader(iter(self.tab2), patterns={3: '0'})
+        self.assertEqual(self.tab2[0], next(fcr))
+        self.assertEqual(self.tab2[4], next(fcr))
+        try:
+            next(fcr)
+            self.fail("Should be no more rows left.")
+        except StopIteration:
+            pass
+
     def test_any_match(self):
         fcr = FilteringCSVReader(iter(self.tab2), patterns={'age': 'only', 0: '2'}, any_match=True)
         self.assertEqual(self.tab2[0], next(fcr))
diff --git a/tests/test_utilities/test_csvsort.py b/tests/test_utilities/test_csvsort.py
index 796d11f..acd5512 100644
--- a/tests/test_utilities/test_csvsort.py
+++ b/tests/test_utilities/test_csvsort.py
@@ -29,7 +29,7 @@ class TestCSVSort(CSVKitTestCase, ColumnsTests, EmptyFileTests, NamesTests):
 
     def test_sort_date(self):
         reader = self.get_output_as_reader(['-c', '2', 'examples/testxls_converted.csv'])
-        test_order = [u'text', u'This row has blanks', u'Unicode! Σ', u'Chicago Tribune', u'Chicago Sun-Times', u'Chicago Reader']
+        test_order = [u'text', u'Chicago Tribune', u'Chicago Sun-Times', u'Chicago Reader', u'This row has blanks', u'Unicode! Σ']
         new_order = [six.text_type(r[0]) for r in reader]
         self.assertEqual(test_order, new_order)
 
@@ -45,8 +45,8 @@ class TestCSVSort(CSVKitTestCase, ColumnsTests, EmptyFileTests, NamesTests):
         new_order = [six.text_type(r[0]) for r in reader]
         self.assertEqual(test_order, new_order)
 
-    def test_sort_ints_and_nulls(self):
+    def test_sort_t_and_nulls(self):
         reader = self.get_output_as_reader(['-c', '2', 'examples/sort_ints_nulls.csv'])
-        test_order = ['b', '', '1', '2']
+        test_order = ['b', '1', '2', '']
         new_order = [six.text_type(r[1]) for r in reader]
         self.assertEqual(test_order, new_order)