Repository: vaexio/vaex

Issue:
cvs reading: copy_index=False should be default
We almost never want it, and when we do, we can pass True


Discussion:

Resolving patch:
diff --git a/packages/vaex-core/vaex/__init__.py b/packages/vaex-core/vaex/__init__.py
--- a/packages/vaex-core/vaex/__init__.py
+++ b/packages/vaex-core/vaex/__init__.py
@@ -106,7 +106,7 @@ def _convert_name(filenames, shuffle=False, suffix=None):
         return base + ".hdf5"
 
 
-def open(path, convert=False, shuffle=False, copy_index=True, *args, **kwargs):
+def open(path, convert=False, shuffle=False, copy_index=False, *args, **kwargs):
     """Open a DataFrame from file given by path.
 
     Example:
@@ -367,7 +367,7 @@ def from_scalars(**kwargs):
     return from_arrays(**{k: np.array([v]) for k, v in kwargs.items()})
 
 
-def from_pandas(df, name="pandas", copy_index=True, index_name="index"):
+def from_pandas(df, name="pandas", copy_index=False, index_name="index"):
     """Create an in memory DataFrame from a pandas DataFrame.
 
     :param: pandas.DataFrame df: Pandas DataFrame
@@ -437,7 +437,7 @@ def from_ascii(path, seperator=None, names=True, skip_lines=0, skip_after=0, **k
     return ds
 
 
-def from_json(path_or_buffer, orient=None, precise_float=False, lines=False, copy_index=True, **kwargs):
+def from_json(path_or_buffer, orient=None, precise_float=False, lines=False, copy_index=False, **kwargs):
     """ A method to read a JSON file using pandas, and convert to a DataFrame directly.
 
     :param str path_or_buffer: a valid JSON string or file-like, default: None
@@ -465,7 +465,7 @@ def from_json(path_or_buffer, orient=None, precise_float=False, lines=False, cop
                        copy_index=copy_index)
 
 
-def from_csv(filename_or_buffer, copy_index=True, chunk_size=None, convert=False, **kwargs):
+def from_csv(filename_or_buffer, copy_index=False, chunk_size=None, convert=False, **kwargs):
     """
     Read a CSV file as a DataFrame, and optionally convert to an hdf5 file.
 
@@ -566,7 +566,7 @@ def read_csv(filepath_or_buffer, **kwargs):
     return from_csv(filepath_or_buffer, **kwargs)
 
 
-def read_csv_and_convert(path, shuffle=False, copy_index=True, **kwargs):
+def read_csv_and_convert(path, shuffle=False, copy_index=False, **kwargs):
     '''Convert a path (or glob pattern) to a single hdf5 file, will open the hdf5 file if exists.
 
     Example:
@@ -575,7 +575,7 @@ def read_csv_and_convert(path, shuffle=False, copy_index=True, **kwargs):
 
     :param str path: path of file or glob pattern for multiple files
     :param bool shuffle: shuffle DataFrame when converting to hdf5
-    :param bool copy_index: by default pandas will create an index (row number), set to false if you want to drop that
+    :param bool copy_index: by default pandas will create an index (row number), set to true if you want to include this as a column.
     :param kwargs: parameters passed to pandas' read_cvs
 
     '''


Test patch:
diff --git a/packages/vaex-core/vaex/test/dataset.py b/packages/vaex-core/vaex/test/dataset.py
--- a/packages/vaex-core/vaex/test/dataset.py
+++ b/packages/vaex-core/vaex/test/dataset.py
@@ -334,10 +334,10 @@ def test_equal(ds1, ds2, units=True, ucds=True, description=True, descriptions=T
 		test_equal(self.dataset, ds2, ucds=False, units=False, description=False, descriptions=False, skip=['m', 'mi'])
 
 		df = self.dataset.to_pandas_df(index_name="name")
-		ds2 = vx.from_pandas(df, index_name="name")
+		ds2 = vx.from_pandas(df, index_name="name", copy_index=True)
 		test_equal(self.dataset, ds2, ucds=False, units=False, description=False, descriptions=False, skip=['m', 'mi'])
 
-		ds2 = vx.from_pandas(self.dataset.to_pandas_df(index_name="name"), copy_index=False)
+		ds2 = vx.from_pandas(self.dataset.to_pandas_df(index_name="name"))
 		assert "name" not in ds2.get_column_names()
 
 		# as astropy table
@@ -387,7 +387,7 @@ def test_csv(self):
 			values = [self.dataset.evaluate(k) for k in 'x y m mi name name_arrow ints f'.split()]
 			for x, y, m, mi, name, name_arrow, i, f_ in zip(*values):#zip(self.x, self.y, self.dataset.data.m, self.dataset.data.mi, self.dataset.data.name, self.dataset.data.ints, self.dataset.data.f):
 				print(separator.join(map(str, [x, y, m, mi, name, name_arrow, i, f_])), file=f)
-		ds = vx.from_csv(fn, index_col=False)
+		ds = vx.from_csv(fn, index_col=False, copy_index=True)
 		changes = self.dataset.compare(ds, report_difference=True)
 		diff = changes[0]
 		#print(diff)
diff --git a/tests/export_test.py b/tests/export_test.py
--- a/tests/export_test.py
+++ b/tests/export_test.py
@@ -44,7 +44,7 @@ def test_export_open_csv(ds_local, tmpdir):
     df = ds_local
     path = str(tmpdir.join('test.csv'))
     df.export_csv(path, chunk_size=3, virtual=True)
-    df_opened = vaex.from_csv(path, copy_index=False)
+    df_opened = vaex.from_csv(path)
     assert list(df) == list(df_opened)
     assert df.shape == df_opened.shape
 
@@ -97,8 +97,8 @@ def test_multi_file_naive_read_convert_export(tmpdir, dtypes):
     pdf1 = pd.read_csv(current_dir + path1, dtype=dtypes)
     pdf2 = pd.read_csv(current_dir + path2, dtype=dtypes)
 
-    vdf1 = vaex.from_pandas(pdf1, copy_index=False)
-    vdf2 = vaex.from_pandas(pdf2, copy_index=False)
+    vdf1 = vaex.from_pandas(pdf1)
+    vdf2 = vaex.from_pandas(pdf2)
 
     magic_value = 1234
     # Verify the Int64 type from pandas is read in vaex correctly
diff --git a/tests/from_csv_test.py b/tests/from_csv_test.py
--- a/tests/from_csv_test.py
+++ b/tests/from_csv_test.py
@@ -14,15 +14,15 @@
 
 def test_from_csv():
     # can read with default options
-    df = vaex.from_csv(csv_path)
+    df = vaex.from_csv(csv_path, copy_index=True)
     _assert_csv_content(df, with_index=True)
 
     # can read an empty CSV
-    df = vaex.from_csv(os.path.join(path, 'data', 'empty.csv'), copy_index=False)
+    df = vaex.from_csv(os.path.join(path, 'data', 'empty.csv'))
     assert len(df) == 0
 
     # can read as chunks iterator
-    df_iterator = vaex.from_csv(csv_path, chunk_size=1, copy_index=False)
+    df_iterator = vaex.from_csv(csv_path, chunk_size=1)
     df1 = next(df_iterator)
     assert len(df1) == 1
     df2, df3 = next(df_iterator), next(df_iterator)
@@ -33,7 +33,7 @@ def test_from_csv():
 
 def test_from_csv_converting_in_chunks():
     # can read several chunks with converting, intermediate files are deleted
-    df = vaex.from_csv(csv_path, copy_index=False, chunk_size=1, convert=True)
+    df = vaex.from_csv(csv_path, chunk_size=1, convert=True)
     _assert_csv_content(df)
     for filename in ['small3.csv_chunk0.hdf5', 'small3.csv_chunk1.hdf5', 'small3.csv_chunk2.hdf5']:
         assert not os.path.exists(os.path.join(path, 'data', filename))
@@ -47,17 +47,17 @@ def test_from_csv_converting_in_chunks():
             vaex.from_csv(f, convert=True)
     with open(csv_path) as f:
         converted_path = os.path.join(path, 'data', 'small3.my.csv.hdf5')
-        df = vaex.from_csv(f, convert=converted_path, copy_index=False)
+        df = vaex.from_csv(f, convert=converted_path)
     _assert_csv_content(df)
     assert os.path.exists(converted_path)
     _cleanup_generated_files(df)
 
     # reuses converted HDF5 file
-    vaex.from_csv(csv_path, copy_index=False, convert=True)
+    vaex.from_csv(csv_path, convert=True)
     assert os.path.exists(os.path.join(path, 'data', 'small3.csv.hdf5'))
     try:
         os.rename(csv_path, csv_path + '_')
-        df = vaex.from_csv(csv_path, copy_index=False, convert=True)
+        df = vaex.from_csv(csv_path, convert=True)
         _assert_csv_content(df)
         _cleanup_generated_files(df)
     except FileNotFoundError as e:
diff --git a/tests/from_json_test.py b/tests/from_json_test.py
--- a/tests/from_json_test.py
+++ b/tests/from_json_test.py
@@ -11,7 +11,7 @@ def test_from_json(ds_local):
     with open(tmp, 'w') as f:
         f.write(pandas_df.to_json())
 
-    tmp_df = vaex.from_json(tmp, copy_index=False)
+    tmp_df = vaex.from_json(tmp)
 
     assert set(tmp_df.column_names) == set(df.column_names)
     assert len(tmp_df) == len(df)
diff --git a/tests/repr_test.py b/tests/repr_test.py
--- a/tests/repr_test.py
+++ b/tests/repr_test.py
@@ -87,7 +87,7 @@ def test_repr_from_pandas():
     # Get pandas dataframe
     dd = pd.DataFrame(dd_dict)
     dd['datetime_7'] = pd.to_timedelta(dd['datetime_2'] - dd['datetime_1'])
-    ds = vaex.from_pandas(dd, copy_index=False)
+    ds = vaex.from_pandas(dd)
     repr_value = repr(ds)
     str_value = str(ds)
     assert 'NaT' in repr_value