Repository: mie-lab/trackintel

Issue:
List of trip ids in a tour dataframe is unstructured when reading it from csv.
similar to  #443 but easier to fix.
Tours store a list of the trips they aggregate in the field "trips". Unfortunately the datatype of this list is not stable when writing+reading it from a csv file.
This can be easily fixed in the function `ti.read_tours_csv` by adding the following line if the column `trips` is present
```python
tours['trips'] = tours['trips'].apply(eval)
```




Comments:

Patch:
diff --git a/environment-dev.yml b/environment-dev.yml
index c1dfc62..0353105 100644
--- a/environment-dev.yml
+++ b/environment-dev.yml
@@ -6,7 +6,7 @@ dependencies:
 - python
 - numpy
 - matplotlib
-- geopandas>=0.10.0
+- geopandas>=0.12.0
 - scikit-learn
 - networkx
 - pint
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 10b132e..5e81a7f 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,6 +1,6 @@
 numpy
 matplotlib
-geopandas>=0.10.0
+geopandas>=0.12.0
 scikit-learn
 networkx 
 pint 
diff --git a/trackintel/io/file.py b/trackintel/io/file.py
index 4d2263d..9e9743a 100644
--- a/trackintel/io/file.py
+++ b/trackintel/io/file.py
@@ -1,3 +1,4 @@
+import ast
 import warnings
 from functools import wraps
 from inspect import signature
@@ -555,6 +556,7 @@ def read_tours_csv(*args, columns=None, index_col=None, tz=None, **kwargs):
     >>> trackintel.read_tours_csv('data.csv', columns={'uuid':'user_id'})
     """
     columns = {} if columns is None else columns
+    kwargs.setdefault("converters", {}).setdefault("trips", ast.literal_eval)
     tours = pd.read_csv(*args, index_col=index_col, **kwargs)
     tours.rename(columns=columns, inplace=True)
 
diff --git a/trackintel/io/postgis.py b/trackintel/io/postgis.py
index 97cb91b..93d98f4 100644
--- a/trackintel/io/postgis.py
+++ b/trackintel/io/postgis.py
@@ -7,6 +7,7 @@ from shapely import wkb
 import pandas as pd
 from geoalchemy2 import Geometry
 from sqlalchemy import create_engine
+from sqlalchemy.types import JSON
 
 import trackintel as ti
 
@@ -50,7 +51,7 @@ def read_positionfixes_postgis(
     parse_dates=None,
     params=None,
     chunksize=None,
-    **kwargs
+    read_gpd_kws=None,
 ):
     """Reads positionfixes from a PostGIS database.
 
@@ -92,7 +93,7 @@ def read_positionfixes_postgis(
         If specified, return an iterator where chunksize is the number
         of rows to include in each chunk.
 
-    **kwargs
+    read_gpd_kws : dict, default None
         Further keyword arguments as available in trackintels trackintel.io.read_positionfixes_gpd().
         Especially useful to rename column names from the SQL table to trackintel conform column names.
         See second example how to use it in code.
@@ -106,7 +107,8 @@ def read_positionfixes_postgis(
     --------
     >>> pfs = ti.io.read_positionfixes_postgis("SELECT * FROM positionfixes", con, geom_col="geom")
     >>> pfs = ti.io.read_positionfixes_postgis("SELECT * FROM positionfixes", con, geom_col="geom",
-    ...                                        index_col="id", user_id="USER", tracked_at="time")
+    ...                                        index_col="id",
+                                               read_gpd_kws={"user_id"="USER", "tracked_at": "time"})
     """
     pfs = gpd.GeoDataFrame.from_postgis(
         sql,
@@ -119,7 +121,7 @@ def read_positionfixes_postgis(
         params=params,
         chunksize=chunksize,
     )
-    return ti.io.read_positionfixes_gpd(pfs, **kwargs)
+    return ti.io.read_positionfixes_gpd(pfs, **(read_gpd_kws or {}))
 
 
 @_handle_con_string
@@ -149,7 +151,7 @@ def read_triplegs_postgis(
     parse_dates=None,
     params=None,
     chunksize=None,
-    **kwargs
+    read_gpd_kws=None,
 ):
     """Reads triplegs from a PostGIS database.
 
@@ -191,7 +193,7 @@ def read_triplegs_postgis(
         If specified, return an iterator where chunksize is the number
         of rows to include in each chunk.
 
-    **kwargs
+    read_gpd_kws : dict, default None
         Further keyword arguments as available in trackintels trackintel.io.read_triplegs_gpd().
         Especially useful to rename column names from the SQL table to trackintel conform column names.
         See second example how to use it in code.
@@ -205,7 +207,7 @@ def read_triplegs_postgis(
     --------
     >>> tpls = ti.io.read_triplegs_postgis("SELECT * FROM triplegs", con, geom_col="geom")
     >>> tpls = ti.io.read_triplegs_postgis("SELECT * FROM triplegs", con, geom_col="geom", index_col="id",
-    ...                                    started_at="start_time", finished_at="end_time", user_id="USER")
+    ...                                    read_gpd_kws={"user_id": "USER"})
     """
     tpls = gpd.GeoDataFrame.from_postgis(
         sql,
@@ -218,7 +220,7 @@ def read_triplegs_postgis(
         params=params,
         chunksize=chunksize,
     )
-    return ti.io.read_triplegs_gpd(tpls, **kwargs)
+    return ti.io.read_triplegs_gpd(tpls, **(read_gpd_kws or {}))
 
 
 @_handle_con_string
@@ -248,7 +250,7 @@ def read_staypoints_postgis(
     parse_dates=None,
     params=None,
     chunksize=None,
-    **kwargs
+    read_gpd_kws=None,
 ):
     """Read staypoints from a PostGIS database.
 
@@ -290,7 +292,7 @@ def read_staypoints_postgis(
         If specified, return an iterator where chunksize is the number
         of rows to include in each chunk.
 
-    **kwargs
+    read_gpd_kws : dict, default None
         Further keyword arguments as available in trackintels trackintel.io.read_staypoints_gpd().
         Especially useful to rename column names from the SQL table to trackintel conform column names.
         See second example how to use it in code.
@@ -305,7 +307,7 @@ def read_staypoints_postgis(
     --------
     >>> sp = ti.io.read_staypoints_postgis("SELECT * FROM staypoints", con, geom_col="geom")
     >>> sp = ti.io.read_staypoints_postgis("SELECT * FROM staypoints", con, geom_col="geom", index_col="id",
-    ...                                      started_at="start_time", finished_at="end_time", user_id="USER")
+    ...                                    read_gpd_kws={"user_id": "USER"})
     """
     sp = gpd.GeoDataFrame.from_postgis(
         sql,
@@ -319,7 +321,7 @@ def read_staypoints_postgis(
         chunksize=chunksize,
     )
 
-    return ti.io.read_staypoints_gpd(sp, **kwargs)
+    return ti.io.read_staypoints_gpd(sp, **(read_gpd_kws or {}))
 
 
 @_handle_con_string
@@ -349,7 +351,8 @@ def read_locations_postgis(
     parse_dates=None,
     params=None,
     chunksize=None,
-    **kwargs
+    extent=None,
+    read_gpd_kws=None,
 ):
     """Reads locations from a PostGIS database.
 
@@ -391,7 +394,10 @@ def read_locations_postgis(
         If specified, return an iterator where chunksize is the number
         of rows to include in each chunk.
 
-    **kwargs
+    extent : string, default None
+        If specified read the extent column as geometry column.
+
+    read_gpd_kws : dict, default None
         Further keyword arguments as available in trackintels trackintel.io.read_locations_gpd().
         Especially useful to rename column names from the SQL table to trackintel conform column names.
         See second example how to use it in code.
@@ -405,7 +411,7 @@ def read_locations_postgis(
     --------
     >>> locs = ti.io.read_locations_postgis("SELECT * FROM locations", con, center="center")
     >>> locs = ti.io.read_locations_postgis("SELECT * FROM locations", con, center="geom", index_col="id",
-    ...                                     user_id="USER", extent="extent")
+    ...                                     extent="extent, read_gpd_kws={"user_id": "USER"})
     )
     """
     locs = gpd.GeoDataFrame.from_postgis(
@@ -419,10 +425,10 @@ def read_locations_postgis(
         params=params,
         chunksize=chunksize,
     )
-    if "extent" in kwargs:
-        locs[kwargs["extent"]] = gpd.GeoSeries.from_wkb(locs[kwargs["extent"]])
+    if extent is not None:
+        locs[extent] = gpd.GeoSeries.from_wkb(locs[extent])
 
-    return ti.io.read_locations_gpd(locs, center=center, **kwargs)
+    return ti.io.read_locations_gpd(locs, center=center, **(read_gpd_kws or {}))
 
 
 @_handle_con_string
@@ -466,7 +472,7 @@ def read_trips_postgis(
     parse_dates=None,
     params=None,
     chunksize=None,
-    **kwargs
+    read_gpd_kws=None,
 ):
     """Read trips from a PostGIS database.
 
@@ -508,7 +514,7 @@ def read_trips_postgis(
         If specified, return an iterator where chunksize is the number
         of rows to include in each chunk.
 
-    **kwargs
+    read_gpd_kws : dict, default None
         Further keyword arguments as available in trackintels trackintel.io.read_trips_gpd().
         Especially useful to rename column names from the SQL table to trackintel conform column names.
         See second example how to use it in code.
@@ -523,8 +529,8 @@ def read_trips_postgis(
     --------
     >>> trips = ti.io.read_trips_postgis("SELECT * FROM trips", con)
     >>> trips = ti.io.read_trips_postgis("SELECT * FROM trips", con, geom_col="geom", index_col="id",
-    ...                                  started_at="start_time", finished_at="end_time", user_id="USER",
-    ...                                  origin_staypoint_id="ORIGIN", destination_staypoint_id="DEST")
+    ...                                  read_gpd_kws={"user_id": "USER", "origin_staypoint_id": "ORIGIN",
+                                                       "destination_staypoint_id": "DEST"})
 
     """
     if geom_col is None:
@@ -550,13 +556,16 @@ def read_trips_postgis(
             chunksize=chunksize,
         )
 
-    return ti.io.read_trips_gpd(trips, **kwargs)
+    return ti.io.read_trips_gpd(trips, **(read_gpd_kws or {}))
 
 
 @_handle_con_string
 def write_trips_postgis(
     trips, name, con, schema=None, if_exists="fail", index=True, index_label=None, chunksize=None, dtype=None
 ):
+    if "trips" in trips.columns:
+        dtype = dtype or {}
+        dtype.setdefault("trips", JSON)
     if isinstance(trips, gpd.GeoDataFrame):
         trips.to_postgis(
             name,
@@ -592,7 +601,7 @@ def read_tours_postgis(
     parse_dates=None,
     params=None,
     chunksize=None,
-    **kwargs
+    read_gpd_kws=None,
 ):
     """Read tours from a PostGIS database.
 
@@ -634,7 +643,7 @@ def read_tours_postgis(
         If specified, return an iterator where chunksize is the number
         of rows to include in each chunk.
 
-    **kwargs
+    read_gpd_kws : dict, default None
         Further keyword arguments as available in trackintels trackintel.io.read_tours_gpd().
         Especially useful to rename column names from the SQL table to trackintel conform column names.
         See second example how to use it in code.
@@ -647,8 +656,8 @@ def read_tours_postgis(
     Examples
     --------
     >>> tours = ti.io.read_tours_postgis("SELECT * FROM tours", con)
-    >>> tours = ti.io.read_tours_postgis("SELECT * FROM tours", con, index_col="id", started_at="start_time",
-    ...                                  finished_at="end_time", user_id="USER")
+    >>> tours = ti.io.read_tours_postgis("SELECT * FROM tours", con, index_col="id",
+                                         read_gpd_kws={"user_id": "USER"})
     """
     if geom_col is None:
         tours = pd.read_sql(
@@ -673,7 +682,7 @@ def read_tours_postgis(
             chunksize=chunksize,
         )
 
-    return ti.io.read_tours_gpd(tours, **kwargs)
+    return ti.io.read_tours_gpd(tours, **(read_gpd_kws or {}))
 
 
 @_handle_con_string


Test patch:
diff --git a/tests/io/test_file.py b/tests/io/test_file.py
index fc78e63..01a50f0 100644
--- a/tests/io/test_file.py
+++ b/tests/io/test_file.py
@@ -2,6 +2,7 @@ import filecmp
 import os
 import pytest
 import pandas as pd
+from pandas.testing import assert_frame_equal
 
 import trackintel as ti
 
@@ -312,6 +313,25 @@ class TestTrips:
         assert gdf.index.name is None
 
 
+@pytest.fixture
+def example_tours():
+    """Tours to load into the database."""
+    t1 = pd.Timestamp("1971-01-01 00:00:00", tz="utc")
+    t2 = pd.Timestamp("1971-01-01 05:00:00", tz="utc")
+    t3 = pd.Timestamp("1971-01-02 07:00:00", tz="utc")
+    h = pd.Timedelta(hours=1)
+
+    list_dict = [
+        {"user_id": 0, "started_at": t1, "finished_at": t1 + h, "trips": [0, 1, 2]},
+        {"user_id": 0, "started_at": t2, "finished_at": t2 + h, "trips": [2, 3, 4]},
+        {"user_id": 1, "started_at": t3, "finished_at": t3 + h, "trips": [4, 5, 6]},
+    ]
+    tours = pd.DataFrame(data=list_dict)
+    tours.index.name = "id"
+    tours.as_tours
+    return tours
+
+
 class TestTours:
     """Test for 'read_tours_csv' and 'write_tours_csv' functions."""
 
@@ -325,6 +345,14 @@ class TestTours:
         assert filecmp.cmp(orig_file, tmp_file, shallow=False)
         os.remove(tmp_file)
 
+    def test_to_from_csv(self, example_tours):
+        """Test writing then reading functionality."""
+        tmp_file = os.path.join("tests", "data", "tours_test.csv")
+        example_tours.as_tours.to_csv(tmp_file)
+        read_tours = ti.read_tours_csv(tmp_file, index_col="id")
+        os.remove(tmp_file)
+        assert_frame_equal(example_tours, read_tours)
+
     def test_to_csv_accessor(self):
         """Test basic reading and writing functions."""
         orig_file = os.path.join("tests", "data", "geolife_long", "tours.csv")
@@ -334,7 +362,3 @@ class TestTours:
         tours.as_tours.to_csv(tmp_file)
         assert filecmp.cmp(orig_file, tmp_file, shallow=False)
         os.remove(tmp_file)
-
-    def test_from_to_postgis(self):
-        # TODO Implement some tests for reading and writing tours.
-        pass
diff --git a/tests/io/test_postgis.py b/tests/io/test_postgis.py
index 021b2b3..74c59cb 100644
--- a/tests/io/test_postgis.py
+++ b/tests/io/test_postgis.py
@@ -312,7 +312,7 @@ class TestPositionfixes:
         try:
             pfs.as_positionfixes.to_postgis(table, conn_string)
             with pytest.warns(UserWarning):
-                pfs_db = ti.io.read_positionfixes_postgis(sql, conn, geom_col, index_col="id", **rename_dict)
+                pfs_db = ti.io.read_positionfixes_postgis(sql, conn, geom_col, index_col="id", read_gpd_kws=rename_dict)
             assert_geodataframe_equal(example_positionfixes, pfs_db)
         finally:
             del_table(conn, table)
@@ -406,7 +406,7 @@ class TestTriplegs:
         try:
             tpls.as_triplegs.to_postgis(table, conn_string)
             with pytest.warns(UserWarning):
-                tpls_db = ti.io.read_triplegs_postgis(sql, conn, geom_col, index_col="id", **rename_dict)
+                tpls_db = ti.io.read_triplegs_postgis(sql, conn, geom_col, index_col="id", read_gpd_kws=rename_dict)
             assert_geodataframe_equal(example_triplegs, tpls_db)
         finally:
             del_table(conn, table)
@@ -571,7 +571,7 @@ class TestLocations:
         try:
             locs.as_locations.to_postgis(table, conn_string)
             with pytest.warns(UserWarning):
-                tpls_db = ti.io.read_locations_postgis(sql, conn, geom_col, index_col="id", **rename_dict)
+                tpls_db = ti.io.read_locations_postgis(sql, conn, geom_col, index_col="id", read_gpd_kws=rename_dict)
             assert_geodataframe_equal(example_locations, tpls_db)
         finally:
             del_table(conn, table)
@@ -624,7 +624,7 @@ class TestTrips:
         try:
             trips.as_trips.to_postgis(table, create_engine(conn_string))
             with pytest.warns(UserWarning):
-                tpls_db = ti.io.read_trips_postgis(sql, conn, index_col="id", **rename_dict)
+                tpls_db = ti.io.read_trips_postgis(sql, conn, index_col="id", read_gpd_kws=rename_dict)
             assert_frame_equal(example_trips, tpls_db)
         finally:
             del_table(conn, table)
@@ -716,6 +716,23 @@ class TestTours:
         finally:
             del_table(conn, table)
 
+    def test_trips_column(self, example_tours, conn_postgis):
+        """Test if list of trips is read correctly."""
+        tours = example_tours
+        tours["trips"] = [[1 + i, 10 + i, 100 + i] for i in range(len(tours))]
+        conn_string, conn = conn_postgis
+        table = "tours"
+        sql = f"SELECT * FROM {table}"
+
+        engine = create_engine(conn_string)
+        try:
+            tours.as_tours.to_postgis(table, engine)
+            with pytest.warns(UserWarning):
+                tours_db = ti.io.read_tours_postgis(sql, conn, index_col="id")
+            assert_frame_equal(tours, tours_db)
+        finally:
+            del_table(conn, table)
+
 
 class TestGetSrid:
     def test_srid(self, example_positionfixes):