Repository: databricks/koalas

Issue:
read_csv(names = ...) inconsistent with pandas
When missing column values for some of the rows, koalas behaves very conservative, whereas pandas just inserts NaNs: 

```python
import pandas as pd
import databricks.koalas as ks

with open("input_file.csv", 'r') as content_file:
    print("Contents of the CSV file")
    print(content_file.read())
    
print("pandas behavior:")
print(pd.read_csv("input_file.csv", names=["A",'B','C','D','E']))

print("koalas behavior:")
print(ks.read_csv("input_file.csv", names=["A",'B','C','D','E']))
```

the output:
```bash
Content of the CSV file
A,B
item1,1
item2,1,2
item3,1,2,3,4
item4,1

pandas behavior:
       A  B    C    D    E
0      A  B  NaN  NaN  NaN
1  item1  1  NaN  NaN  NaN
2  item2  1  2.0  NaN  NaN
3  item3  1  2.0  3.0  4.0
4  item4  1  NaN  NaN  NaN
koalas behavior:
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-12-1acfe46dedcd> in <module>
     13 
     14 print("koalas behavior:")
---> 15 print(ks.read_csv("input_file.csv", names=["A",'B','C','D','E']))

/miniconda3/lib/python3.7/site-packages/databricks/koalas/namespace.py in read_csv(path, header, names, usecols, mangle_dupe_cols, parse_dates, comment)
    197                 raise ValueError('Found non-unique column index')
    198             if len(names) != len(sdf.schema):
--> 199                 raise ValueError('Names do not match the number of columns: %d' % len(names))
    200             sdf = sdf.selectExpr(*["`%s` as `%s`" % (field.name, name)
    201                                    for field, name in zip(sdf.schema, names)])

ValueError: Names do not match the number of columns: 5
```


Discussion:
Actually, this isn't easy to fix as it seems. Spark's CSV schema inference does not support incremental field addition in the middle. Looks we might be able to work around via using RDD API with `spark.read.csv(csv: RDD)` but it will degrade performance and somewhat hacky.

For now, I will simply add Spark schema support into `names` parameters to provide workaround to users for now.

Resolving patch:
diff --git a/databricks/koalas/namespace.py b/databricks/koalas/namespace.py
--- a/databricks/koalas/namespace.py
+++ b/databricks/koalas/namespace.py
@@ -135,9 +135,11 @@ def read_csv(path, header='infer', names=None, usecols=None,
         the first line of the file, if column names are passed explicitly then
         the behavior is identical to `header=None`. Explicitly pass `header=0` to be
         able to replace existing names
-    names : array-like, optional
+    names : str or array-like, optional
         List of column names to use. If file contains no header row, then you should
         explicitly pass `header=None`. Duplicates in this list will cause an error to be issued.
+        If a string is given, it should be a DDL-formatted string in Spark SQL, which is
+        preferred to avoid schema inference for better performance.
     usecols : list-like or callable, optional
         Return a subset of the columns. If list-like, all elements must either be
         positional (i.e. integer indices into the document columns) or strings that
@@ -191,17 +193,21 @@ def read_csv(path, header='infer', names=None, usecols=None,
                 raise ValueError("Only length-1 comment characters supported")
             reader.option("comment", comment)
 
-        sdf = reader.csv(path)
-
-        if header is None:
-            sdf = sdf.selectExpr(*["`%s` as `%s`" % (field.name, i)
-                                   for i, field in enumerate(sdf.schema)])
-        if names is not None:
+        if isinstance(names, str):
+            sdf = reader.schema(names).csv(path)
+        else:
+            sdf = reader.csv(path)
+            if header is None:
+                sdf = sdf.selectExpr(*["`%s` as `%s`" % (field.name, i)
+                                       for i, field in enumerate(sdf.schema)])
+        if isinstance(names, list):
             names = list(names)
             if len(set(names)) != len(names):
                 raise ValueError('Found non-unique column index')
             if len(names) != len(sdf.schema):
-                raise ValueError('Names do not match the number of columns: %d' % len(names))
+                raise ValueError('The number of names [%s] does not match the number '
+                                 'of columns [%d]. Try names by a Spark SQL DDL-formatted '
+                                 'string.' % (len(sdf.schema), len(names)))
             sdf = sdf.selectExpr(*["`%s` as `%s`" % (field.name, name)
                                    for field, name in zip(sdf.schema, names)])
 


Test patch:
diff --git a/databricks/koalas/tests/test_csv.py b/databricks/koalas/tests/test_csv.py
--- a/databricks/koalas/tests/test_csv.py
+++ b/databricks/koalas/tests/test_csv.py
@@ -49,6 +49,17 @@ def csv_text(self):
             Edith,600
             """)
 
+    @property
+    def csv_text_2(self):
+        return normalize_text(
+            """
+            A,B
+            item1,1
+            item2,1,2
+            item3,1,2,3,4
+            item4,1
+            """)
+
     @property
     def csv_text_with_comments(self):
         return normalize_text(
@@ -101,15 +112,21 @@ def check(header='infer', names=None, usecols=None):
 
             self.assertRaisesRegex(ValueError, 'non-unique',
                                    lambda: koalas.read_csv(fn, names=['n', 'n']))
-            self.assertRaisesRegex(ValueError, 'Names do not match.*3',
+            self.assertRaisesRegex(ValueError, 'does not match the number.*3',
                                    lambda: koalas.read_csv(fn, names=['n', 'a', 'b']))
-            self.assertRaisesRegex(ValueError, 'Names do not match.*3',
+            self.assertRaisesRegex(ValueError, 'does not match the number.*3',
                                    lambda: koalas.read_csv(fn, header=0, names=['n', 'a', 'b']))
             self.assertRaisesRegex(ValueError, 'Usecols do not match.*3',
                                    lambda: koalas.read_csv(fn, usecols=[1, 3]))
             self.assertRaisesRegex(ValueError, 'Usecols do not match.*col',
                                    lambda: koalas.read_csv(fn, usecols=['amount', 'col']))
 
+    def test_read_with_spark_schema(self):
+        with self.csv_file(self.csv_text_2) as fn:
+            actual = koalas.read_csv(fn, names="A string, B string, C long, D long, E long")
+            expected = pd.read_csv(fn, names=['A', 'B', 'C', 'D', 'E'])
+            self.assertEqual(repr(expected), repr(actual))
+
     def test_read_csv_with_comment(self):
         with self.csv_file(self.csv_text_with_comments) as fn:
             expected = pd.read_csv(fn, comment='#')