Repository: databricks/koalas

Issue:
pandas.read_csv
Implement a `pyspark.read_csv` method with the following options:
 - `index_col` -> depends on the design of indexing, so let's leave it out for now
 - `columns`
 - `mangle_dupe_cols`
 - `parse_dates`
 - `comment`
 - `usecols`

Documentation:
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_csv.html

Dask implementation:
https://github.com/dask/dask/blob/master/dask/dataframe/io/csv.py

Dask test suite:
https://github.com/dask/dask/blob/master/dask/dataframe/io/tests/test_csv.py



Discussion:

Resolving patch:
diff --git a/pandorable_sparky/namespace.py b/pandorable_sparky/namespace.py
--- a/pandorable_sparky/namespace.py
+++ b/pandorable_sparky/namespace.py
@@ -1,6 +1,8 @@
 """
 Wrappers around spark that correspond to common pandas functions.
 """
+import sys
+
 import pyspark.sql
 import numpy as np
 import pandas as pd
@@ -8,20 +10,106 @@
 from pyspark.sql import Column
 from pyspark.sql.types import StructType
 
+if sys.version > '3':
+    basestring = unicode = str
+
 
 def default_session():
     return pyspark.sql.SparkSession.builder.getOrCreate()
 
 
-def read_csv(path, header='infer'):
-    b = default_session().read.format("csv").option("inferSchema", "true")
-    if header == 'infer':
-        b = b.option("header", "true")
-    elif header == 0:
-        pass
+def read_csv(path, header='infer', names=None, usecols=None,
+             mangle_dupe_cols=True, parse_dates=False, comment=None):
+    """Read CSV (comma-separated) file into DataFrame.
+
+    :param path: The path string storing the CSV file to be read.
+    :param header: Whether to to use as the column names, and the start of the data.
+                   Default behavior is to infer the column names: if no names are passed
+                   the behavior is identical to `header=0` and column names are inferred from
+                   the first line of the file, if column names are passed explicitly then
+                   the behavior is identical to `header=None`. Explicitly pass `header=0` to be
+                   able to replace existing names
+    :param names: List of column names to use. If file contains no header row, then you should
+                  explicitly pass `header=None`. Duplicates in this list will cause an error to be
+                  issued.
+    :param usecols: Return a subset of the columns. If list-like, all elements must either be
+                    positional (i.e. integer indices into the document columns) or strings that
+                    correspond to column names provided either by the user in names or inferred
+                    from the document header row(s).
+                    If callable, the callable function will be evaluated against the column names,
+                    returning names where the callable function evaluates to `True`.
+    :param mangle_dupe_cols: Duplicate columns will be specified as 'X0', 'X1', ... 'XN', rather
+                             than 'X' ... 'X'. Passing in False will cause data to be overwritten if
+                             there are duplicate names in the columns.
+                             Currently only `True` is allowed.
+    :param parse_dates: boolean or list of ints or names or list of lists or dict, default `False`.
+                        Currently only `False` is allowed.
+    :param comment: Indicates the line should not be parsed.
+    :return: :class:`DataFrame`
+    """
+    if mangle_dupe_cols is not True:
+        raise ValueError("mangle_dupe_cols can only be `True`: %s" % mangle_dupe_cols)
+    if parse_dates is not False:
+        raise ValueError("parse_dates can only be `False`: %s" % parse_dates)
+
+    if usecols is not None and not callable(usecols):
+        usecols = list(usecols)
+    if usecols is None or callable(usecols) or len(usecols) > 0:
+        reader = default_session().read.option("inferSchema", "true")
+
+        if header == 'infer':
+            header = 0 if names is None else None
+        if header == 0:
+            reader.option("header", True)
+        elif header is None:
+            reader.option("header", False)
+        else:
+            raise ValueError("Unknown header argument {}".format(header))
+
+        if comment is not None:
+            if not isinstance(comment, basestring) or len(comment) != 1:
+                raise ValueError("Only length-1 comment characters supported")
+            reader.option("comment", comment)
+
+        df = reader.csv(path)
+
+        if header is None:
+            df = df.selectExpr(*["`%s` as `%s`" % (field.name, i)
+                                 for i, field in enumerate(df.schema)])
+        if names is not None:
+            names = list(names)
+            if len(set(names)) != len(names):
+                raise ValueError('Found non-unique column index')
+            if len(names) != len(df.schema):
+                raise ValueError('Names do not match the number of columns: %d' % len(names))
+            df = df.selectExpr(*["`%s` as `%s`" % (field.name, name)
+                                 for field, name in zip(df.schema, names)])
+
+        if usecols is not None:
+            if callable(usecols):
+                cols = [field.name for field in df.schema if usecols(field.name)]
+                missing = []
+            elif all(isinstance(col, int) for col in usecols):
+                cols = [field.name for i, field in enumerate(df.schema) if i in usecols]
+                missing = [col for col in usecols
+                           if col >= len(df.schema) or df.schema[col].name not in cols]
+            elif all(isinstance(col, basestring) for col in usecols):
+                cols = [field.name for field in df.schema if field.name in usecols]
+                missing = [col for col in usecols if col not in cols]
+            else:
+                raise ValueError("'usecols' must either be list-like of all strings, "
+                                 "all unicode, all integers or a callable.")
+            if len(missing) > 0:
+                raise ValueError('Usecols do not match columns, columns expected but not found: %s'
+                                 % missing)
+
+            if len(cols) > 0:
+                df = df.select(cols)
+            else:
+                df = default_session().createDataFrame([], schema=StructType())
     else:
-        raise ValueError("Unknown header argument {}".format(header))
-    return b.load(path)
+        df = default_session().createDataFrame([], schema=StructType())
+    return df
 
 
 def read_parquet(path, columns=None):


Test patch:
diff --git a/pandorable_sparky/testing/utils.py b/pandorable_sparky/testing/utils.py
--- a/pandorable_sparky/testing/utils.py
+++ b/pandorable_sparky/testing/utils.py
@@ -146,14 +146,28 @@ def assertPandasEqual(self, expected, result):
                "\n\nResult:\n%s\n%s" % (result, result.dtypes))
         self.assertTrue(expected.equals(result), msg=msg)
 
+    def assertPandasAlmostEqual(self, expected, result):
+        msg = ("DataFrames are not equal: " +
+               "\n\nExpected:\n%s\n%s" % (expected, expected.dtypes) +
+               "\n\nResult:\n%s\n%s" % (result, result.dtypes))
+        self.assertEqual(expected.shape, result.shape, msg=msg)
+        for ecol, rcol in zip(expected.columns, result.columns):
+            self.assertEqual(str(ecol), str(rcol), msg=msg)
+            for eval, rval in zip(expected[ecol], result[rcol]):
+                self.assertAlmostEqual(eval, rval, msg=msg)
+
 
 class TestUtils(object):
 
     @contextmanager
     def temp_dir(self):
         tmp = tempfile.mkdtemp()
-        shutil.rmtree(tmp)
         try:
             yield tmp
         finally:
             shutil.rmtree(tmp)
+
+    @contextmanager
+    def temp_file(self):
+        with self.temp_dir() as tmp:
+            yield tempfile.mktemp(dir=tmp)
diff --git a/pandorable_sparky/tests/test_csv.py b/pandorable_sparky/tests/test_csv.py
new file mode 100644
--- /dev/null
+++ b/pandorable_sparky/tests/test_csv.py
@@ -0,0 +1,126 @@
+from contextlib import contextmanager
+from distutils.version import LooseVersion
+import unittest
+
+import pandas as pd
+import pandorable_sparky
+import pyspark
+
+from pandorable_sparky.testing.utils import ReusedSQLTestCase, TestUtils
+
+
+def normalize_text(s):
+    return '\n'.join(map(str.strip, s.strip().split('\n')))
+
+
+class CsvTest(ReusedSQLTestCase, TestUtils):
+
+    @property
+    def csv_text(self):
+        return normalize_text(
+            """
+            name,amount
+            Alice,100
+            Bob,-200
+            Charlie,300
+            Dennis,400
+            Edith,-500
+            Frank,600
+            Alice,200
+            Frank,-200
+            Bob,600
+            Alice,400
+            Frank,200
+            Alice,300
+            Edith,600
+            """)
+
+    @property
+    def csv_text_with_comments(self):
+        return normalize_text(
+            """
+            # header
+            %s
+            # comment
+            Alice,400
+            Edith,600
+            # footer
+            """ % self.csv_text)
+
+    @contextmanager
+    def csv_file(self, csv):
+        with self.temp_file() as tmp:
+            with open(tmp, 'w') as f:
+                f.write(csv)
+            yield tmp
+
+    def test_read_csv(self):
+        with self.csv_file(self.csv_text) as fn:
+
+            def check(header='infer', names=None, usecols=None):
+                expected = pd.read_csv(fn, header=header, names=names, usecols=usecols)
+                actual = pyspark.read_csv(fn, header=header, names=names, usecols=usecols)
+                self.assertPandasAlmostEqual(expected, actual.toPandas())
+
+            check()
+            check(header=None)
+            check(header=0)
+            check(names=['n', 'a'])
+            check(header=0, names=['n', 'a'])
+            check(usecols=[1])
+            check(usecols=[1, 0])
+            check(usecols=['amount'])
+            check(usecols=['amount', 'name'])
+            if LooseVersion("0.20.0") <= LooseVersion(pd.__version__):
+                check(usecols=lambda x: x == 'amount')
+            check(usecols=[])
+            check(usecols=[1, 1])
+            check(usecols=['amount', 'amount'])
+            if LooseVersion("0.20.0") <= LooseVersion(pd.__version__):
+                check(usecols=lambda x: x == 'a')
+            check(names=['n', 'a'], usecols=['a'])
+
+            self.assertRaisesRegex(ValueError, 'non-unique',
+                                   lambda: pyspark.read_csv(fn, names=['n', 'n']))
+            self.assertRaisesRegex(ValueError, 'Names do not match.*3',
+                                   lambda: pyspark.read_csv(fn, names=['n', 'a', 'b']))
+            self.assertRaisesRegex(ValueError, 'Names do not match.*3',
+                                   lambda: pyspark.read_csv(fn, header=0, names=['n', 'a', 'b']))
+            self.assertRaisesRegex(ValueError, 'Usecols do not match.*3',
+                                   lambda: pyspark.read_csv(fn, usecols=[1, 3]))
+            self.assertRaisesRegex(ValueError, 'Usecols do not match.*col',
+                                   lambda: pyspark.read_csv(fn, usecols=['amount', 'col']))
+
+    def test_read_csv_with_comment(self):
+        with self.csv_file(self.csv_text_with_comments) as fn:
+            expected = pd.read_csv(fn, comment='#')
+            actual = pyspark.read_csv(fn, comment='#')
+            self.assertPandasAlmostEqual(expected, actual.toPandas())
+
+            self.assertRaisesRegex(ValueError, 'Only length-1 comment characters supported',
+                                   lambda: pyspark.read_csv(fn, comment='').show())
+            self.assertRaisesRegex(ValueError, 'Only length-1 comment characters supported',
+                                   lambda: pyspark.read_csv(fn, comment='##').show())
+            self.assertRaisesRegex(ValueError, 'Only length-1 comment characters supported',
+                                   lambda: pyspark.read_csv(fn, comment=1))
+            self.assertRaisesRegex(ValueError, 'Only length-1 comment characters supported',
+                                   lambda: pyspark.read_csv(fn, comment=[1]))
+
+    def test_read_csv_with_mangle_dupe_cols(self):
+        self.assertRaisesRegex(ValueError, 'mangle_dupe_cols',
+                               lambda: pyspark.read_csv('path', mangle_dupe_cols=False))
+
+    def test_read_csv_with_parse_dates(self):
+        self.assertRaisesRegex(ValueError, 'parse_dates',
+                               lambda: pyspark.read_csv('path', parse_dates=True))
+
+
+if __name__ == "__main__":
+    from pandorable_sparky.tests.test_csv import *
+
+    try:
+        import xmlrunner
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/pandorable_sparky/tests/test_parquet.py b/pandorable_sparky/tests/test_parquet.py
--- a/pandorable_sparky/tests/test_parquet.py
+++ b/pandorable_sparky/tests/test_parquet.py
@@ -20,7 +20,7 @@ def test_local(self):
                 'bhello': np.random.choice(['hello', 'yo', 'people'], size=1000).astype("O")})
             data = data[['i32', 'i64', 'f', 'bhello']]
             self.spark.createDataFrame(data, 'i32 int, i64 long, f double, bhello string') \
-                .coalesce(1).write.parquet(tmp)
+                .coalesce(1).write.parquet(tmp, mode='overwrite')
 
             def check(columns, expected):
                 if LooseVersion("0.21.1") <= LooseVersion(pd.__version__):