Repository: wireservice/csvkit

Issue:
csvstack [Errno 24] Too many open files
`csvstack` returns "[Errno 24] Too many open files" when many files are added for stacking. For example, `csvstack --filenames *.csv > stacked.csv`. It seems to be a Python error, but can `csvstack` use an intermediate file and close the previous ones?


Comments:

Patch:
diff --git a/.travis.yml b/.travis.yml
index ca53492..d5a2d31 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,10 +7,8 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
-  - "pypy-5.3.1"
 install:
   - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; else pip install -r requirements-py2.txt; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == "pypy-5.3.1" ]]; then pip install psycopg2cffi; else pip install psycopg2; fi
   - pip install coveralls
 before_script:
   - psql -U postgres -c 'CREATE DATABASE dummy_test'
diff --git a/AUTHORS.rst b/AUTHORS.rst
index ce1a210..ad1e207 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -81,3 +81,4 @@ The following individuals have contributed code to csvkit:
 * kjedamzik
 * John Vandenberg
 * Olivier Lacan
+* Adrien Delessert
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index ea9460b..0cf8633 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -3,14 +3,19 @@
 
 Improvements:
 
+* :doc:`/scripts/csvgrep` adds a :code:`--any-match` (:code:`-a`) flag to select rows where any column matches instead of all columns.
 * :doc:`/scripts/csvjson` no longer emits a property if its value is null.
 * :doc:`/scripts/in2csv` adds a :code:`--encoding-xls` option to specify the encoding of the input XLS file.
 
 Fixes:
 
 * :doc:`/scripts/csvgrep` accepts utf-8 arguments to the :code:`--match` and :code:`--regex` options in Python 2.
+* :doc:`/scripts/csvsql` sets a DECIMAL's precision and scale and a VARCHAR's length to avoid dialect-specific errors.
+* :doc:`/scripts/csvstack` no longer opens all files at once.
 * :doc:`/scripts/in2csv` respects :code:`--no-header-row` when :code:`--no-inference` is set.
 
+csvkit is no longer tested on PyPy.
+
 1.0.2 - April 28, 2017
 ----------------------
 
diff --git a/csvkit/utilities/csvgrep.py b/csvkit/utilities/csvgrep.py
index df18eb8..c0b080b 100644
--- a/csvkit/utilities/csvgrep.py
+++ b/csvkit/utilities/csvgrep.py
@@ -35,7 +35,8 @@ class CSVGrep(CSVKitUtility):
                                     help='If specified, must be the path to a file. For each tested row, if any line in the file (stripped of line separators) is an exact match for the cell value, the row will pass.')
         self.argparser.add_argument('-i', '--invert-match', dest='inverse', action='store_true',
                                     help='If specified, select non-matching instead of matching rows.')
-
+        self.argparser.add_argument('-a', '--any-match', dest='any_match', action='store_true',
+                                    help='If specified, select rows where any column matches instead of all columns.')
     def main(self):
         if self.args.names_only:
             self.print_column_names()
@@ -67,7 +68,7 @@ class CSVGrep(CSVKitUtility):
             pattern = self.args.pattern
 
         patterns = dict((column_id, pattern) for column_id in column_ids)
-        filter_reader = FilteringCSVReader(rows, header=False, patterns=patterns, inverse=self.args.inverse)
+        filter_reader = FilteringCSVReader(rows, header=False, patterns=patterns, inverse=self.args.inverse, any_match=self.args.any_match)
 
         output = agate.csv.writer(self.output_file, **writer_kwargs)
         output.writerow(column_names)
diff --git a/csvkit/utilities/csvstack.py b/csvkit/utilities/csvstack.py
index 56e2051..cba10af 100644
--- a/csvkit/utilities/csvstack.py
+++ b/csvkit/utilities/csvstack.py
@@ -23,20 +23,15 @@ class CSVStack(CSVKitUtility):
                                     help='Use the filename of each input file as its grouping value. When specified, -g will be ignored.')
 
     def main(self):
-        self.input_files = []
-
-        for path in self.args.input_paths:
-            self.input_files.append(self._open_input_file(path))
-
-        if not self.input_files:
+        if not self.args.input_paths:
             self.argparser.error('You must specify at least one file to stack.')
 
-        if self.args.group_by_filenames:
-            groups = [os.path.basename(f.name) for f in self.input_files]
-        elif self.args.groups:
+        has_groups = self.args.group_by_filenames or self.args.groups
+
+        if self.args.groups and not self.args.group_by_filenames:
             groups = self.args.groups.split(',')
 
-            if len(groups) != len(self.input_files):
+            if len(groups) != len(self.args.input_paths):
                 self.argparser.error('The number of grouping values must be equal to the number of CSV files being stacked.')
         else:
             groups = None
@@ -45,7 +40,9 @@ class CSVStack(CSVKitUtility):
 
         output = agate.csv.writer(self.output_file, **self.writer_kwargs)
 
-        for i, f in enumerate(self.input_files):
+        for i, path in enumerate(self.args.input_paths):
+            f = self._open_input_file(path)
+
             if isinstance(self.args.skip_lines, int):
                 skip_lines = self.args.skip_lines
                 while skip_lines > 0:
@@ -56,12 +53,18 @@ class CSVStack(CSVKitUtility):
 
             rows = agate.csv.reader(f, **self.reader_kwargs)
 
+            if has_groups:
+                if groups:
+                    group = groups[i]
+                else:
+                    group = os.path.basename(f.name)
+
             # If we have header rows, use them
             if not self.args.no_header_row:
                 headers = next(rows, [])
 
                 if i == 0:
-                    if groups:
+                    if has_groups:
                         headers.insert(0, group_name)
 
                     output.writerow(headers)
@@ -72,19 +75,19 @@ class CSVStack(CSVKitUtility):
                 headers = make_default_headers(len(row))
 
                 if i == 0:
-                    if groups:
+                    if has_groups:
                         headers.insert(0, group_name)
 
                     output.writerow(headers)
 
-                if groups:
-                    row.insert(0, groups[i])
+                if has_groups:
+                    row.insert(0, group)
 
                 output.writerow(row)
 
             for row in rows:
-                if groups:
-                    row.insert(0, groups[i])
+                if has_groups:
+                    row.insert(0, group)
 
                 output.writerow(row)
 
diff --git a/docs/scripts/csvgrep.rst b/docs/scripts/csvgrep.rst
index 7998053..d40b6d1 100644
--- a/docs/scripts/csvgrep.rst
+++ b/docs/scripts/csvgrep.rst
@@ -38,6 +38,8 @@ Filter tabular data to only those rows where certain columns contain a given val
                             row will pass.
       -i, --invert-match    If specified, select non-matching instead of matching
                             rows.
+      -a  --any-match       If specified, select rows where any column matches
+                            instead of all columns.
 
 See also: :doc:`../common_arguments`.
 
@@ -53,8 +55,7 @@ Search for the row relating to Illinois::
 Search for rows relating to states with names beginning with the letter "I"::
 
     csvgrep -c 1 -r "^I" examples/realdata/FY09_EDU_Recipients_by_State.csv
-    
+
 Search for rows that do not contain an empty state cell::
 
     csvgrep -c 1 -r "^$" -i examples/realdata/FY09_EDU_Recipients_by_State.csv
-


Test patch:
diff --git a/tests/test_utilities/test_csvgrep.py b/tests/test_utilities/test_csvgrep.py
index 08b59d1..a4c8eb2 100644
--- a/tests/test_utilities/test_csvgrep.py
+++ b/tests/test_utilities/test_csvgrep.py
@@ -33,6 +33,12 @@ class TestCSVGrep(CSVKitTestCase, ColumnsTests, EmptyFileTests, NamesTests):
             ['1', '2', '3'],
         ])
 
+    def test_any_match(self):
+        self.assertRows(['-c', '1,2,3', '-a', '-m', '1', 'examples/dummy.csv'], [
+            ['a', 'b', 'c'],
+            ['1', '2', '3'],
+        ])
+
     def test_match_utf8(self):
         self.assertRows(['-c', '3', '-m', 'ʤ', 'examples/test_utf8.csv'], [
             ['foo', 'bar', 'baz'],