Repository: chezou/tabula-py (python)

Issue:
Column header
# Summary of your issue

I am extracting data from a table using the "guess=True" option. Unfortunately, the first row is imported as column header. (The guess is not really wrong, since the typeface is bold and there is a line below it, see [Example](https://esv.ch/documents/ranglisten/2017-06-05_12_Schwingertag_Laupen-ST.pdf).) I didn't find I way to tell read_pdf_table not to treat the particular first line as column header. 

Would it be possible to add a "header=None" option?



Pull request:
Add pandas options fix #38
This PR add `pandas_options` for `read_pdf()` method.

NOTE: Using `multiple_tables` options changes using pandas method internally: from `pd.read_csv()` to `pd.DataFrame()`
So, it might be raise error with passing same `pandas_options` with/without `multiple_tables`.

Patch:
diff --git a/tabula/wrapper.py b/tabula/wrapper.py
index b2bb1fe..c1e6407 100644
--- a/tabula/wrapper.py
+++ b/tabula/wrapper.py
@@ -22,7 +22,12 @@ JAR_NAME = "tabula-0.9.2-jar-with-dependencies.jar"
 JAR_DIR = os.path.abspath(os.path.dirname(__file__))
 JAR_PATH = os.path.join(JAR_DIR, JAR_NAME)
 
-def read_pdf(input_path, **kwargs):
+def read_pdf(input_path,
+             output_format='dataframe',
+             encoding='utf-8',
+             java_options=None,
+             pandas_options=None,
+             multiple_tables=False, **kwargs):
     '''Read tables in PDF.
 
     Args:
@@ -34,6 +39,12 @@ def read_pdf(input_path, **kwargs):
             Encoding type for pandas. Default is 'utf-8'
         java_options (list, optional):
             Set java options like `-Xmx256m`.
+        pandas_options (dict, optional):
+            Set pandas options like {'header': None}.
+        multiple_tables (bool, optional):
+            This is experimental option. It enables to handle multple tables within a page.
+            Note: If `multiple_tables` option is enabled, tabula-py uses not `pd.read_csv()`,
+             but `pd.DataFrame()`. Make sure to pass appropreate `pandas_options`.
         kwargs (dict):
             Dictionary of option for tabula-java. Details are shown in `build_options()`
 
@@ -41,21 +52,19 @@ def read_pdf(input_path, **kwargs):
         Extracted pandas DataFrame or list.
     '''
 
-    output_format = kwargs.get('output_format', 'dataframe')
-
     if output_format == 'dataframe':
         kwargs.pop('format', None)
 
     elif output_format == 'json':
         kwargs['format'] = 'JSON'
 
-    multiple_tables = kwargs.get('multiple_tables')
     if multiple_tables:
-        kwargs.pop('multiple_tables', None)
         kwargs['format'] = 'JSON'
 
-    java_options = kwargs.get('java_options', [])
-    if isinstance(java_options, str):
+    if java_options is None:
+        java_options = []
+
+    elif isinstance(java_options, str):
         java_options = [java_options]
 
     options = build_options(kwargs)
@@ -71,25 +80,28 @@ def read_pdf(input_path, **kwargs):
     if len(output) == 0:
         return
 
-    encoding = kwargs.get('encoding', 'utf-8')
+    if pandas_options is None:
+        pandas_options = {}
 
     fmt = kwargs.get('format')
     if fmt == 'JSON':
         if multiple_tables:
-            return extract_from(json.loads(output.decode(encoding)))
+            return extract_from(json.loads(output.decode(encoding)), pandas_options)
 
         else:
             return json.loads(output.decode(encoding))
 
     else:
-        return pd.read_csv(io.BytesIO(output), encoding=encoding)
+        pandas_options['encoding'] = pandas_options.get('encoding', encoding)
+
+        return pd.read_csv(io.BytesIO(output), **pandas_options)
 
 
 # Set alias for future rename from `read_pdf_table` to `read_pdf`
 read_pdf_table = deprecated(read_pdf)
 
 
-def convert_into(input_path, output_path, **kwargs):
+def convert_into(input_path, output_path, output_format='csv', java_options=None, **kwargs):
     '''Convert tables from PDF into a file.
 
     Args:
@@ -98,7 +110,7 @@ def convert_into(input_path, output_path, **kwargs):
         output_path (str):
             File path of output file.
         output_format (str, optional):
-            Output format of this function (csv, json or tsv)
+            Output format of this function (csv, json or tsv). Default: csv
         java_options (list, optional):
             Set java options like `-Xmx256m`.
         kwargs (dict):
@@ -112,10 +124,12 @@ def convert_into(input_path, output_path, **kwargs):
         raise AttributeError("'output_path' shoud not be None or empty")
 
     kwargs['output_path'] = output_path
-    kwargs['format'] = extract_format_for_conversion(kwargs.get('output_format'))
+    kwargs['format'] = extract_format_for_conversion(output_format)
+
+    if java_options is None:
+        java_options = []
 
-    java_options = kwargs.get('java_options', [])
-    if isinstance(java_options, str):
+    elif isinstance(java_options, str):
         java_options = [java_options]
 
     options = build_options(kwargs)
@@ -128,7 +142,7 @@ def convert_into(input_path, output_path, **kwargs):
         if is_url:
             os.unlink(path)
 
-def convert_into_by_batch(input_dir, **kwargs):
+def convert_into_by_batch(input_dir, output_format='csv', java_options=None, **kwargs):
     '''Convert tables from PDFs in a directory.
 
     Args:
@@ -148,10 +162,12 @@ def convert_into_by_batch(input_dir, **kwargs):
     if input_dir is None or not os.path.isdir(input_dir):
         raise AttributeError("'input_dir' shoud be directory path")
 
-    kwargs['format'] = extract_format_for_conversion(kwargs.get('output_format'))
+    kwargs['format'] = extract_format_for_conversion(output_format)
 
-    java_options = kwargs.get('java_options', [])
-    if isinstance(java_options, str):
+    if java_options is None:
+        java_options = []
+
+    elif isinstance(java_options, str):
         java_options = [java_options]
 
     # Option for batch
@@ -178,22 +194,56 @@ def extract_format_for_conversion(output_format='csv'):
         raise AttributeError("'output_format' has no attribute 'dataframe'")
 
 
-def extract_from(raw_json):
+def extract_from(raw_json, pandas_options=None):
     '''Extract tables from json.
 
     Args:
         raw_json (list):
             Decoded list from tabula-java JSON.
+        pandas_options (dict optional):
+            pandas options for `pd.DataFrame()`
     '''
 
     data_frames = []
+    if pandas_options is None:
+        pandas_options = {}
+
+    columns = pandas_options.pop('columns', None)
+    columns, header_line_number = convert_pandas_csv_options(pandas_options, columns)
 
     for table in raw_json:
         list_data = [[np.nan if not e['text'] else e['text'] for e in row] for row in table['data']]
-        data_frames.append(pd.DataFrame(list_data))
+        _columns = columns
+
+        if isinstance(header_line_number, int) and not columns:
+            _columns = list_data.pop(header_line_number)
+            _columns = ['' if e is np.nan else e for e in _columns]
+
+        data_frames.append(pd.DataFrame(data=list_data, columns=_columns, **pandas_options))
 
     return data_frames
 
+def convert_pandas_csv_options(pandas_options, columns):
+    ''' Translate `pd.read_csv()` options into `pd.DataFrame()` especially for header.
+
+    Args:
+        pandas_option (dict):
+            pandas options like {'header': None}.
+        columns (list):
+            list of column name.
+    '''
+
+    _columns = pandas_options.pop('names', columns)
+    header = pandas_options.pop('header', None)
+
+    if header == 'infer':
+        header_line_number = 0 if not bool(_columns) else None
+    else:
+        header_line_number = header
+
+    return _columns, header_line_number
+
+
 def localize_file(path):
     '''Ensure localize target file.
 
@@ -222,7 +272,7 @@ def localize_file(path):
         return path, is_url
 
 
-def build_options(kwargs={}):
+def build_options(kwargs=None):
     '''Build options for tabula-java
 
     Args:
@@ -264,6 +314,8 @@ def build_options(kwargs={}):
         Built dictionary of options
     '''
     __options = []
+    if kwargs is None:
+        kwargs = {}
     options = kwargs.get('options', '')
     # handle options described in string for backward compatibility
     __options += shlex.split(options)


Test patch:
diff --git a/tests/test_read_pdf_table.py b/tests/test_read_pdf_table.py
index 1113f6a..12ec7d8 100644
--- a/tests/test_read_pdf_table.py
+++ b/tests/test_read_pdf_table.py
@@ -46,6 +46,35 @@ class TestReadPdfTable(unittest.TestCase):
         self.assertTrue(tabula.read_pdf(pdf_path, pages=1, java_options=['-Xmx256m']
                                        ).equals(pd.read_csv(expected_csv1)))
 
+    def test_read_pdf_with_pandas_option(self):
+        pdf_path = 'tests/resources/data.pdf'
+        expected_csv1 = 'tests/resources/data_1.csv'
+        column_name = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+        self.assertTrue(tabula.read_pdf(pdf_path, pages=1, pandas_options={'header': None}
+                                       ).equals(pd.read_csv(expected_csv1, header=None)))
+        self.assertTrue(tabula.read_pdf(pdf_path, pages=1, pandas_options={'header': 0}
+                                       ).equals(pd.read_csv(expected_csv1, header=0)))
+        self.assertTrue(tabula.read_pdf(pdf_path, pages=1, pandas_options={'header': 'infer'}
+                                       ).equals(pd.read_csv(expected_csv1, header='infer')))
+        self.assertTrue(
+            tabula.read_pdf(
+                pdf_path, pages=1, pandas_options={'header': 'infer', 'names': column_name}
+                ).equals(pd.read_csv(expected_csv1, header='infer', names=column_name))
+        )
+        self.assertTrue(
+            tabula.read_pdf(
+                pdf_path, pages=1, multiple_tables=True,
+                pandas_options={'header': 'infer', 'names': column_name}
+                )[0].equals(pd.read_csv(expected_csv1, header='infer', names=column_name))
+        )
+        self.assertTrue(
+            tabula.read_pdf(
+                pdf_path, pages=1, multiple_tables=True,
+                pandas_options={'header': 'infer', 'columns': column_name}
+                )[0].equals(pd.read_csv(expected_csv1, header='infer', names=column_name))
+        )
+
+
     def test_read_pdf_for_multiple_tables(self):
         pdf_path = 'tests/resources/data.pdf'
         expected_csv1 = 'tests/resources/data_1.csv'