Repository: wireservice/csvkit

Issue:
csvstat: flag to specify how many frequent values to display
Csvstat is really a very useful tool, but it still could be better : the "frequent values" feature is apparently fixedly limited to 5 values. It would be great if this "5" value was only the default value, and could be altered by a new parameter, say "-f ". 

For example : "csvstat -f 20 -c 3,7 --freq myfile.csv" would return the 20 most frequent values of colums 3 and 7. Today it (seems) impossible to exceed the limit of 5 values.

Best regards, thanks for this great kit.


Discussion:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 6ebf3be..06441ce 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -6,7 +6,8 @@ Improvements:
 * Add a :code:`--version` (:code:`-V`) flag.
 * :code:`-I` is the short option for :code:`--no-inference`.
 * :doc:`/scripts/csvjoin` supports :code:`--snifflimit` and :code:`--no-inference`.
-* :doc:`/scripts/in2csv` now supports a :code:`--names` flag to print Excel sheet names.
+* :doc:`/scripts/csvstat` adds a :code:`--freq-count` option to set the maximum number of frequent values to display.
+* :doc:`/scripts/in2csv` adds a :code:`--names` flag to print Excel sheet names.
 
 Fixes:
 
diff --git a/csvkit/utilities/csvstat.py b/csvkit/utilities/csvstat.py
index 2292e77..29908d2 100644
--- a/csvkit/utilities/csvstat.py
+++ b/csvkit/utilities/csvstat.py
@@ -13,9 +13,6 @@ from csvkit.cli import CSVKitUtility, parse_column_identifiers
 
 NoneType = type(None)
 
-MAX_UNIQUE = 5
-MAX_FREQ = 5
-
 OPERATIONS = OrderedDict([
     ('type', {
         'aggregation': None,
@@ -97,8 +94,10 @@ class CSVStat(CSVKitUtility):
                                     help='Only output the length of the longest values.')
         self.argparser.add_argument('--freq', dest='freq_only', action='store_true',
                                     help='Only output lists of frequent values.')
+        self.argparser.add_argument('--freq-count', dest='freq_count', type=int,
+                                    help='The maximum number of frequent values to display.')
         self.argparser.add_argument('--count', dest='count_only', action='store_true',
-                                    help='Only output total row count')
+                                    help='Only output total row count.')
         self.argparser.add_argument('-y', '--snifflimit', dest='sniff_limit', type=int,
                                     help='Limit CSV dialect sniffing to the specified number of bytes. Specify "0" to disable sniffing entirely.')
 
@@ -144,18 +143,23 @@ class CSVStat(CSVKitUtility):
             self.get_column_offset()
         )
 
+        kwargs = {}
+
+        if self.args.freq_count:
+            kwargs['freq_count'] = self.args.freq_count
+
         # Output a single stat
         if operations:
             if len(column_ids) == 1:
-                self.print_one(table, column_ids[0], operations[0], label=False)
+                self.print_one(table, column_ids[0], operations[0], label=False, **kwargs)
             else:
                 for column_id in column_ids:
-                    self.print_one(table, column_id, operations[0])
+                    self.print_one(table, column_id, operations[0], **kwargs)
         else:
             stats = {}
 
             for column_id in column_ids:
-                stats[column_id] = self.calculate_stats(table, column_id)
+                stats[column_id] = self.calculate_stats(table, column_id, **kwargs)
 
             # Output as CSV
             if self.args.csv_output:
@@ -164,7 +168,7 @@ class CSVStat(CSVKitUtility):
             else:
                 self.print_stats(table, column_ids, stats)
 
-    def print_one(self, table, column_id, operation, label=True):
+    def print_one(self, table, column_id, operation, label=True, **kwargs):
         """
         Print data for a single statistic.
         """
@@ -178,7 +182,7 @@ class CSVStat(CSVKitUtility):
 
             try:
                 if getter:
-                    stat = getter(table, column_id)
+                    stat = getter(table, column_id, **kwargs)
                 else:
                     op = OPERATIONS[op_name]['aggregation']
                     stat = table.aggregate(op(column_id))
@@ -198,7 +202,7 @@ class CSVStat(CSVKitUtility):
         else:
             self.output_file.write(u'%s\n' % stat)
 
-    def calculate_stats(self, table, column_id):
+    def calculate_stats(self, table, column_id, **kwargs):
         """
         Calculate stats for all valid operations.
         """
@@ -212,7 +216,7 @@ class CSVStat(CSVKitUtility):
 
                 try:
                     if getter:
-                        stats[op_name] = getter(table, column_id)
+                        stats[op_name] = getter(table, column_id, **kwargs)
                     else:
                         op = op_data['aggregation']
                         v = table.aggregate(op(column_id))
@@ -314,16 +318,16 @@ class CSVStat(CSVKitUtility):
             writer.writerow(output_row)
 
 
-def get_type(table, column_id):
+def get_type(table, column_id, **kwargs):
     return '%s' % table.columns[column_id].data_type.__class__.__name__
 
 
-def get_unique(table, column_id):
+def get_unique(table, column_id, **kwargs):
     return len(table.columns[column_id].values_distinct())
 
 
-def get_freq(table, column_id):
-    return table.pivot(column_id).order_by('Count', reverse=True).limit(MAX_FREQ)
+def get_freq(table, column_id, freq_count=5, **kwargs):
+    return table.pivot(column_id).order_by('Count', reverse=True).limit(freq_count)
 
 
 def launch_new_instance():


Test patch:
diff --git a/tests/test_utilities/test_csvstat.py b/tests/test_utilities/test_csvstat.py
index 2f9cec2..875c7dd 100644
--- a/tests/test_utilities/test_csvstat.py
+++ b/tests/test_utilities/test_csvstat.py
@@ -55,6 +55,14 @@ class TestCSVStat(CSVKitTestCase, ColumnsTests, EmptyFileTests, NamesTests):
         self.assertIn('SALINE (59x)', output)
         self.assertNotIn('MIAMI (56x)', output)
 
+
+    def test_freq_count(self):
+        output = self.get_output(['examples/realdata/ks_1033_data.csv', '--freq-count', '1'])
+
+        self.assertIn('WYANDOTTE (123x)', output)
+        self.assertNotIn('SALINE (59x)', output)
+        self.assertNotIn('MIAMI (56x)', output)
+
     def test_csv(self):
         output = self.get_output_as_io(['--csv', 'examples/realdata/ks_1033_data.csv'])