Repository: mie-lab/trackintel
Pull request #100 (2021-02-07 18:12:16)

Feat: flexible csv column names
Allows the user to rename csv columns directly while using the e.g. read_triplegs_csv functions
modified tests accordingly
added test data with modified column names


Patch:
diff --git a/trackintel/io/file.py b/trackintel/io/file.py
index ba78852..5446510 100644
--- a/trackintel/io/file.py
+++ b/trackintel/io/file.py
@@ -10,6 +10,11 @@ def read_positionfixes_csv(*args, **kwargs):
     builds a geopandas GeoDataFrame. This also validates that the ingested data
     conforms to the trackintel understanding of positionfixes (see 
     :doc:`/modules/model`). 
+    
+    Parameters
+    ----------
+    columns : dict
+        The columnnames to rename in the format {'old_name':'trackintel_standard_name'}.
 
     Note that this function is primarily useful if data is available in a 
     longitude/latitude format. If your data already contains a WKT column, it
@@ -23,8 +28,11 @@ def read_positionfixes_csv(*args, **kwargs):
     Examples
     --------
     >>> trackintel.read_positionfixes_csv('data.csv')
+    >>> trackintel.read_positionfixes_csv('data.csv', columns={'time':'tracked_at', 'User':'user_id'})
     """
+    columns=kwargs.pop('columns',{})
     df = pd.read_csv(*args, **kwargs)
+    df = df.rename(columns=columns)
     df['geom'] = list(zip(df.longitude, df.latitude))
     df['geom'] = df['geom'].apply(Point)
     df['tracked_at'] = df['tracked_at'].apply(dateutil.parser.parse)
@@ -58,12 +66,25 @@ def read_triplegs_csv(*args, **kwargs):
     builds a geopandas GeoDataFrame. This also validates that the ingested data
     conforms to the trackintel understanding of triplegs (see :doc:`/modules/model`).
 
+
+   Parameters
+    ----------
+    columns : dict
+        The columnnames to rename in the format {'old_name':'trackintel_standard_name'}.
+    
     Returns
     -------
     GeoDataFrame
         A GeoDataFrame containing the triplegs.
+        
+    Examples
+    --------
+    >>> trackintel.read_triplegs_csv('data.csv')
+    >>> trackintel.read_triplegs_csv('data.csv', columns={'start_time':'started_at', 'User':'user_id'})
     """
+    columns=kwargs.pop('columns',{})
     df = pd.read_csv(*args, **kwargs)
+    df = df.rename(columns=columns)
     df['geom'] = df['geom'].apply(wkt.loads)
     df['started_at'] = df['started_at'].apply(dateutil.parser.parse)
     df['finished_at'] = df['finished_at'].apply(dateutil.parser.parse)
@@ -72,6 +93,7 @@ def read_triplegs_csv(*args, **kwargs):
     return gdf
 
 
+
 def write_triplegs_csv(triplegs, filename, *args, **kwargs):
     """Wraps the pandas to_csv function, but transforms the geom into WKT 
     before writing.
@@ -94,13 +116,25 @@ def read_staypoints_csv(*args, **kwargs):
     geometry and builds a geopandas GeoDataFrame. This also validates that 
     the ingested data conforms to the trackintel understanding of staypoints 
     (see :doc:`/modules/model`).
+    
+    Parameters
+    ----------
+    columns : dict
+        The columnnames to rename in the format {'old_name':'trackintel_standard_name'}.
 
     Returns
     -------
     GeoDataFrame
         A GeoDataFrame containing the staypoints.
+            
+    Examples
+    --------
+    >>> trackintel.read_staypoints_csv('data.csv')
+    >>> trackintel.read_staypoints_csv('data.csv', columns={'start_time':'started_at', 'User':'user_id'})
     """
+    columns=kwargs.pop('columns',{})
     df = pd.read_csv(*args, **kwargs)
+    df = df.rename(columns=columns)
     df['geom'] = df['geom'].apply(wkt.loads)
     df['started_at'] = df['started_at'].apply(dateutil.parser.parse)
     df['finished_at'] = df['finished_at'].apply(dateutil.parser.parse)
@@ -132,13 +166,25 @@ def read_locations_csv(*args, **kwargs):
     validates that the ingested data conforms to the trackintel understanding 
     of locations (see :doc:`/modules/model`).
 
+    Parameters
+    ----------
+    columns : dict
+        The columnnames to rename in the format {'old_name':'trackintel_standard_name'}.
+
     Returns
     -------
     GeoDataFrame
         A GeoDataFrame containing the locations.
+        
+            
+    Examples
+    --------
+    >>> trackintel.read_locations_csv('data.csv')
+    >>> trackintel.read_locations_csv('data.csv', columns={'start_time':'started_at', 'User':'user_id'})
     """
-    # Todo: How to implement flexible geometry names in locations (gdf with potentially 2 geometry columns)
+    columns=kwargs.pop('columns',{})
     df = pd.read_csv(*args, **kwargs)
+    df = df.rename(columns=columns)
     df['center'] = df['center'].apply(wkt.loads)
     if 'extent' in df.columns:
         df['extent'] = df['extent'].apply(wkt.loads)
@@ -170,19 +216,33 @@ def read_trips_csv(*args, **kwargs):
     """Wraps the pandas read_csv function and extraces proper datetimes. This also 
     validates that the ingested data conforms to the trackintel understanding 
     of trips (see :doc:`/modules/model`).
+    
+    Parameters
+    ----------
+    columns : dict
+        The columnnames to rename in the format {'old_name':'trackintel_standard_name'}.
 
     Returns
     -------
     DataFrame
         A DataFrame containing the trips.
+        
+            
+    Examples
+    --------
+    >>> trackintel.read_trips_csv('data.csv')
+    >>> trackintel.read_trips_csv('data.csv', columns={'start_time':'started_at', 'User':'user_id'})
     """
+    columns=kwargs.pop('columns',{})
     df = pd.read_csv(*args, **kwargs)
+    df = df.rename(columns=columns)
     df['started_at'] = df['started_at'].apply(dateutil.parser.parse)
     df['finished_at'] = df['finished_at'].apply(dateutil.parser.parse)
     assert df.as_trips
     return df
 
 
+
 def write_trips_csv(trips, filename, *args, **kwargs):
     """Wraps the pandas to_csv function.
 


Test patch:
diff --git a/tests/data/locations_mod_columns.csv b/tests/data/locations_mod_columns.csv
new file mode 100644
index 0000000..b072cdc
--- /dev/null
+++ b/tests/data/locations_mod_columns.csv
@@ -0,0 +1,3 @@
+;user_id;elevation;geom;extent
+0;1;5;POINT (8.5487794879999992 47.3765150500000018);POLYGON ((8.5487794879999992 47.3765150500000018, 8.5276999999999994 47.3993499999999983, 8.5116899999999998 47.4085400000000021, 8.5487794879999992 47.3765150500000018))
+1;1;5;POINT (8.5633999999999997 47.9560000000000031);POLYGON ((8.5633999999999997 47.9560000000000031, 8.6456000000000000 47.2334499999999977, 8.4534000000000002 47.2633999999999972, 8.5633999999999997 47.9560000000000031))
\ No newline at end of file
diff --git a/tests/data/positionfixes_mod_columns.csv b/tests/data/positionfixes_mod_columns.csv
new file mode 100644
index 0000000..08106de
--- /dev/null
+++ b/tests/data/positionfixes_mod_columns.csv
@@ -0,0 +1,7 @@
+;user_id;time;lat;lon;elevation;accuracy
+0;1;2015-11-27T08:00:00Z;47.37651505;8.548779488;456;1
+1;1;2015-11-27T08:20:22Z;47.39935;8.5277;524;1
+2;1;2015-11-27T12:39:28Z;47.40854;8.51169;537;1
+3;1;2015-11-27T13:57:20Z;47.41144;8.54472;447;1
+4;1;2015-11-28T12:39:28Z;43.40854;8.51169;537;1
+5;1;2015-11-28T13:57:20Z;43.41144;8.54472;447;1
\ No newline at end of file
diff --git a/tests/data/staypoints_mod_columns.csv b/tests/data/staypoints_mod_columns.csv
new file mode 100644
index 0000000..fdbe0eb
--- /dev/null
+++ b/tests/data/staypoints_mod_columns.csv
@@ -0,0 +1,3 @@
+;User;started_at;finished_at;elevation;geom
+0;1;2015-11-27T08:00:00Z;2015-11-27T10:00:00Z;350;POINT (8.5282230000000006 47.3951860000000025)
+1;1;2015-11-27T12:00:00Z;2015-11-27T14:00:00Z;370;POINT (8.5434000000000001 47.9560000000000031)
\ No newline at end of file
diff --git a/tests/data/triplegs_mod_columns.csv b/tests/data/triplegs_mod_columns.csv
new file mode 100644
index 0000000..cb4e60d
--- /dev/null
+++ b/tests/data/triplegs_mod_columns.csv
@@ -0,0 +1,3 @@
+;user_id;start_time;end_time;tripleg
+0;1;2015-11-27T08:00:00Z;2015-11-27T10:00:00Z;LINESTRING (8.5487794879999992 47.3765150500000018, 8.5276999999999994 47.3993499999999983, 8.5116899999999998 47.4085400000000021)
+1;1;2015-11-27T12:00:00Z;2015-11-27T14:00:00Z;LINESTRING (8.5633999999999997 47.9560000000000031, 8.6456000000000000 47.2334499999999977, 8.4534000000000002 47.2633999999999972)
\ No newline at end of file
diff --git a/tests/data/trips_mod_columns.csv b/tests/data/trips_mod_columns.csv
new file mode 100644
index 0000000..53f6016
--- /dev/null
+++ b/tests/data/trips_mod_columns.csv
@@ -0,0 +1,5 @@
+;user_id;started_at;finished_at;orig_stp;dest_stp
+0;1;2015-11-27T08:00:00Z;2015-11-27T08:15:00Z;2;5
+1;1;2015-11-27T08:20:22Z;2015-11-27T08:35:22Z;2;5
+2;1;2015-11-27T12:39:28Z;2015-11-27T12:55:28Z;2;5
+3;1;2015-11-27T13:57:20Z;2015-11-27T14:13:20Z;2;5
\ No newline at end of file
diff --git a/tests/io/test_file.py b/tests/io/test_file.py
index faa6742..ae69801 100644
--- a/tests/io/test_file.py
+++ b/tests/io/test_file.py
@@ -10,8 +10,11 @@ import pandas as pd
 class TestFile:
     def test_positionfixes_from_to_csv(self):
         orig_file = os.path.join('tests', 'data', 'positionfixes.csv')
+        mod_file = os.path.join('tests','data', 'positionfixes_mod_columns.csv')
         tmp_file = os.path.join('tests', 'data', 'positionfixes_test.csv')
         pfs = ti.read_positionfixes_csv(orig_file, sep=';')
+        mod_pfs = ti.read_positionfixes_csv(mod_file, columns={'lat':'latitude', 'lon':'longitude', 'time':'tracked_at'},sep=';')
+        assert mod_pfs.equals(pfs)
         pfs['tracked_at'] = pfs['tracked_at'].apply(lambda d: d.isoformat().replace('+00:00', 'Z'))
         pfs.as_positionfixes.to_csv(tmp_file, sep=';',
                                     columns=['user_id', 'tracked_at', 'latitude', 'longitude', 'elevation', 'accuracy'])
@@ -26,8 +29,11 @@ class TestFile:
 
     def test_triplegs_from_to_csv(self):
         orig_file = os.path.join('tests', 'data', 'triplegs.csv')
+        mod_file = os.path.join('tests','data','triplegs_mod_columns.csv')
         tmp_file = os.path.join('tests', 'data', 'triplegs_test.csv')
         tpls = ti.read_triplegs_csv(orig_file, sep=';')
+        mod_tpls = ti.read_triplegs_csv(mod_file, columns={'start_time':'started_at','end_time':'finished_at', 'tripleg':'geom'},sep=';')
+        assert mod_tpls.equals(tpls)
         tpls['started_at'] = tpls['started_at'].apply(lambda d: d.isoformat().replace('+00:00', 'Z'))
         tpls['finished_at'] = tpls['finished_at'].apply(lambda d: d.isoformat().replace('+00:00', 'Z'))
         tpls.as_triplegs.to_csv(tmp_file, sep=';',
@@ -44,8 +50,11 @@ class TestFile:
 
     def test_staypoints_from_to_csv(self):
         orig_file = os.path.join('tests', 'data', 'staypoints.csv')
+        mod_file = os.path.join('tests', 'data', 'staypoints_mod_columns.csv')
         tmp_file = os.path.join('tests', 'data', 'staypoints_test.csv')
         stps = ti.read_staypoints_csv(orig_file, sep=';')
+        mod_stps = ti.read_staypoints_csv(mod_file, columns={'User':'user_id'},sep=';')
+        assert mod_stps.equals(stps)
         stps['started_at'] = stps['started_at'].apply(lambda d: d.isoformat().replace('+00:00', 'Z'))
         stps['finished_at'] = stps['finished_at'].apply(lambda d: d.isoformat().replace('+00:00', 'Z'))
         stps.as_staypoints.to_csv(tmp_file, sep=';',
@@ -61,13 +70,17 @@ class TestFile:
 
     def test_locations_from_to_csv(self):
         orig_file = os.path.join('tests', 'data', 'locations.csv')
+        mod_file = os.path.join('tests','data','locations_mod_columns.csv')
         tmp_file = os.path.join('tests', 'data', 'locations_test.csv')
+        mod_plcs = ti.read_locations_csv(mod_file, columns={'geom':'center'},sep=';')
         locs = ti.read_locations_csv(orig_file, sep=';')
+        assert mod_plcs.equals(locs)
         locs.as_locations.to_csv(tmp_file, sep=';',
-                                 columns=['user_id', 'elevation', 'center', 'extent'])
+                              columns=['user_id', 'elevation', 'center', 'extent'])
         assert filecmp.cmp(orig_file, tmp_file, shallow=False)
         os.remove(tmp_file)
         
+        
     def test_locations_from_to_postgis(self):
         # TODO Implement some tests for PostGIS.
         pass
@@ -76,8 +89,11 @@ class TestFile:
 
     def test_trips_from_to_csv(self):
         orig_file = os.path.join('tests', 'data', 'trips.csv')
+        mod_file = os.path.join('tests', 'data', 'trips_mod_columns.csv')
         tmp_file = os.path.join('tests', 'data', 'trips_test.csv')
         tpls = ti.read_trips_csv(orig_file, sep=';')
+        mod_tpls = ti.read_trips_csv(mod_file, columns= {'orig_stp':'origin_staypoint_id','dest_stp':'destination_staypoint_id'},sep=';')
+        assert mod_tpls.equals(tpls)
         tpls['started_at'] = tpls['started_at'].apply(lambda d: d.isoformat().replace('+00:00', 'Z'))
         tpls['finished_at'] = tpls['finished_at'].apply(lambda d: d.isoformat().replace('+00:00', 'Z'))
         tpls.as_trips.to_csv(tmp_file, sep=';',