Repository: databricks/koalas

Issue:
Implement read_table and DataFrame.to_table
Generic functionality to read from / write to a Spark SQL table.


Discussion:

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -1509,6 +1509,51 @@ def to_koalas(self):
         else:
             return DataFrame(self)
 
+    def to_table(self, name: str, format: Optional[str] = None, mode: str = 'error',
+                 partition_cols: Union[str, List[str], None] = None,
+                 **options):
+        """
+        Write the DataFrame into a Spark table.
+
+        Parameters
+        ----------
+        name : str, required
+            Table name in Spark.
+        format : string, optional
+            Specifies the output data source format. Some common ones are:
+
+            - 'delta'
+            - 'parquet'
+            - 'orc'
+            - 'json'
+            - 'csv'
+
+        mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'}, default 'error'.
+            Specifies the behavior of the save operation when the table exists already.
+
+            - 'append': Append the new data to existing data.
+            - 'overwrite': Overwrite existing data.
+            - 'ignore': Silently ignore this operation if data already exists.
+            - 'error' or 'errorifexists': Throw an exception if data already exists.
+
+        partition_cols : str or list of str, optional, default None
+            Names of partitioning columns
+        options
+            Additional options passed directly to Spark.
+
+        See Also
+        --------
+        read_table
+        DataFrame.to_spark_io
+        DataFrame.to_parquet
+
+        Examples
+        --------
+        >>> df.to_table('my_database.my_table', partition_cols='date')  # doctest: +SKIP
+        """
+        self._sdf.write.saveAsTable(name=name, format=format, mode=mode,
+                                    partitionBy=partition_cols, options=options)
+
     def to_parquet(self, path: str, mode: str = 'error',
                    partition_cols: Union[str, List[str], None] = None,
                    compression: Optional[str] = None):
@@ -1520,7 +1565,7 @@ def to_parquet(self, path: str, mode: str = 'error',
         path : str, required
             Path to write to.
         mode : str {'append', 'overwrite', 'ignore', 'error', 'errorifexists'}, default 'error'.
-            Specifies the behavior of the save operation when data already.
+            Specifies the behavior of the save operation when the destination exists already.
 
             - 'append': Append the new data to existing data.
             - 'overwrite': Overwrite existing data.
@@ -1536,12 +1581,14 @@ def to_parquet(self, path: str, mode: str = 'error',
         See Also
         --------
         read_parquet
+        DataFrame.to_table
+        DataFrame.to_spark_io
 
         Examples
         --------
-        >>> df.write.parquet('my_data.parquet', partition='date')  # doctest: +SKIP
+        >>> df.to_parquet('my_data.parquet', partition_cols='date')  # doctest: +SKIP
 
-        >>> df.write.parquet('my_data.parquet', partition=['date', 'country'])  # doctest: +SKIP
+        >>> df.to_parquet('my_data.parquet', partition_cols=['date', 'country'])  # doctest: +SKIP
         """
         self._sdf.write.parquet(path=path, mode=mode, partitionBy=partition_cols,
                                 compression=compression)
diff --git a/databricks/koalas/namespace.py b/databricks/koalas/namespace.py
--- a/databricks/koalas/namespace.py
+++ b/databricks/koalas/namespace.py
@@ -227,6 +227,33 @@ def read_csv(path, header='infer', names=None, usecols=None,
     return DataFrame(sdf)
 
 
+def read_table(name: str) -> DataFrame:
+    """
+    Read a Spark table and return a DataFrame.
+
+    Parameters
+    ----------
+    name : string
+        Table name in Spark.
+
+    Returns
+    -------
+    DataFrame
+
+    See Also
+    --------
+    DataFrame.to_table
+    read_spark_io
+    read_parquet
+
+    Examples
+    --------
+    >>> ks.read_table('my_database.my_table')  # doctest: +SKIP
+    """
+    sdf = default_session().read.table(name)
+    return DataFrame(sdf)
+
+
 def read_spark_io(path: Optional[str] = None, format: Optional[str] = None,
                   schema: Union[str, 'StructType'] = None, **options) -> DataFrame:
     """Load a DataFrame from a Spark data source.
@@ -253,6 +280,7 @@ def read_spark_io(path: Optional[str] = None, format: Optional[str] = None,
     See Also
     --------
     DataFrame.to_spark_io
+    DataFrame.read_table
     DataFrame.read_parquet
 
     Examples
@@ -263,7 +291,7 @@ def read_spark_io(path: Optional[str] = None, format: Optional[str] = None,
     return DataFrame(sdf)
 
 
-def read_parquet(path, columns=None):
+def read_parquet(path, columns=None) -> DataFrame:
     """Load a parquet object from the file path, returning a DataFrame.
 
     Parameters


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe_spark_io.py b/databricks/koalas/tests/test_dataframe_spark_io.py
--- a/databricks/koalas/tests/test_dataframe_spark_io.py
+++ b/databricks/koalas/tests/test_dataframe_spark_io.py
@@ -89,6 +89,24 @@ def test_parquet_write(self):
             actual = ks.read_parquet(tmp)[self.test_column_order]
             self.assert_eq(actual.sort_values(by='f'), expected.sort_values(by='f'))
 
+    def test_table(self):
+        pdf = self.test_pdf
+        expected = ks.DataFrame(pdf)
+
+        # Write out partitioned by one column
+        expected.to_table('test_table', mode='overwrite', partition_cols='i32')
+        # Reset column order, as once the data is written out, Spark rearranges partition
+        # columns to appear first.
+        actual = ks.read_table('test_table')[self.test_column_order]
+        self.assert_eq(actual.sort_values(by='f'), expected.sort_values(by='f'))
+
+        # Write out partitioned by two columns
+        expected.to_table('test_table', mode='overwrite', partition_cols=['i32', 'bhello'])
+        # Reset column order, as once the data is written out, Spark rearranges partition
+        # columns to appear first.
+        actual = ks.read_table('test_table')[self.test_column_order]
+        self.assert_eq(actual.sort_values(by='f'), expected.sort_values(by='f'))
+
     def test_spark_io(self):
         with self.temp_dir() as tmp:
             pdf = self.test_pdf