Repository: mie-lab/trackintel
Pull request #387 (2022-04-02 10:16:39)

ENH: Replace df.append with pd.concat (Future warning)
In future versions of Pandas, `concat` will be deprecated. 
Read more about this [here](https://github.com/pandas-dev/pandas/issues/35407).


Patch:
diff --git a/.github/workflows/rstcheck.yml b/.github/workflows/rstcheck.yml
index d8b7f71..3c87ffd 100644
--- a/.github/workflows/rstcheck.yml
+++ b/.github/workflows/rstcheck.yml
@@ -19,7 +19,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python -m pip install sphinx==3.2.1 rstcheck==3.3.1
+          python -m pip install jinja2==3.0.3 sphinx==3.2.1 rstcheck==3.3.1
 
       - name: Run rstcheck
         run: |
diff --git a/trackintel/analysis/location_identification.py b/trackintel/analysis/location_identification.py
index 41867bc..33e691c 100644
--- a/trackintel/analysis/location_identification.py
+++ b/trackintel/analysis/location_identification.py
@@ -306,8 +306,12 @@ def osna_method(staypoints):
 
     # create a pivot table -> labels "home" and "work" as columns. ("user_id", "location_id" still in index.)
     sp_pivot = sp_agg.unstack()
+    # this line is here to circumvent a bug in pandas .idxmax should return NA if all values in column are
+    # empty but that doesn't work for pd.NaT -> therefor we cast to float64
+    sp_pivot /= pd.Timedelta("1ns")
     # get index of maximum for columns "work" and "home"
-    sp_idxmax = sp_pivot.groupby(["user_id"]).idxmax()
+    # looks over locations to find maximum for columns
+    sp_idxmax = sp_pivot.groupby(["user_id"]).idxmax(axis="index")
     # first assign labels
     for col in sp_idxmax.columns:
         sp_pivot.loc[sp_idxmax[col].dropna(), "purpose"] = col
@@ -317,7 +321,7 @@ def osna_method(staypoints):
     if all(col in sp_idxmax.columns for col in ["work", "home"]):
         redo_work = sp_idxmax[sp_idxmax["home"] == sp_idxmax["work"]]
         sp_pivot.loc[redo_work["work"], "purpose"] = "home"
-        sp_pivot.loc[redo_work["work"], "work"] = pd.NaT
+        sp_pivot.loc[redo_work["work"], "work"] = np.nan
         sp_idxmax_work = sp_pivot.groupby(["user_id"])["work"].idxmax()
         sp_pivot.loc[sp_idxmax_work.dropna(), "purpose"] = "work"
 
diff --git a/trackintel/analysis/tracking_quality.py b/trackintel/analysis/tracking_quality.py
index fe0781b..254b9c6 100644
--- a/trackintel/analysis/tracking_quality.py
+++ b/trackintel/analysis/tracking_quality.py
@@ -262,7 +262,7 @@ def _split_overlaps(source, granularity="day", max_iter=60):
         new_df.loc[change_flag, "started_at"] = df.loc[change_flag, "finished_at"]
         new_df.loc[change_flag, "finished_at"] = finished_at_temp
 
-        df = df.append(new_df, ignore_index=True, sort=True)
+        df = pd.concat((df, new_df), ignore_index=True, sort=True)
 
         change_flag = __get_split_index(df, granularity=granularity)
         iter_count += 1
diff --git a/trackintel/preprocessing/util.py b/trackintel/preprocessing/util.py
index ebcafb3..5813430 100644
--- a/trackintel/preprocessing/util.py
+++ b/trackintel/preprocessing/util.py
@@ -131,6 +131,6 @@ def _explode_agg(column, agg, orig_df, agg_df):
         Original Dataframe with additional colum from aggregated DataFrame.
     """
     temp = agg_df.explode(column)
-    temp.index = temp[column]
     temp = temp[temp[column].notna()]
+    temp.index = temp[column]
     return orig_df.join(temp[agg], how="left")


Test patch:
diff --git a/tests/analysis/test_location_identification.py b/tests/analysis/test_location_identification.py
index 3cbfe50..a256ca5 100644
--- a/tests/analysis/test_location_identification.py
+++ b/tests/analysis/test_location_identification.py
@@ -308,7 +308,7 @@ class TestOsna_Method:
         ]
         sp = gpd.GeoDataFrame(data=list_dict, geometry="geom", crs="EPSG:4326")
         sp.index.name = "id"
-        sp = example_osna.append(sp)
+        sp = pd.concat((example_osna, sp))
 
         result = osna_method(sp).iloc[:-2]
         example_osna.loc[example_osna["location_id"] == 0, "purpose"] = "home"
@@ -339,7 +339,7 @@ class TestOsna_Method:
 
     def test_two_users(self, example_osna):
         """Test if two users are handled correctly."""
-        two_user = example_osna.append(example_osna)
+        two_user = pd.concat((example_osna, example_osna))
         two_user.iloc[len(example_osna) :, 0] = 1  # second user gets id 1
         result = osna_method(two_user)
         two_user.loc[two_user["location_id"] == 0, "purpose"] = "home"
@@ -418,22 +418,22 @@ class TestOsna_Method:
         example_osna.loc[example_osna["location_id"] == 1, "purpose"] = "work"
         assert_geodataframe_equal(example_osna, result)
 
-    # def test_multiple_users_with_only_one_location(self):
-    #     """Test that function can handle multiple users with only one location."""
-    #     t_leis = pd.Timestamp("2021-07-14 01:00:00", tz="utc")
-    #     t_work = pd.Timestamp("2021-07-14 18:00:00", tz="utc")
-    #     h = pd.Timedelta("1h")
-    #     list_dict = [
-    #         {"user_id": 0, "location_id": 0, "started_at": t_leis, "finished_at": t_leis + h},
-    #         {"user_id": 0, "location_id": 1, "started_at": t_work, "finished_at": t_work + h},
-    #         {"user_id": 1, "location_id": 0, "started_at": t_leis, "finished_at": t_leis + h},
-    #         {"user_id": 2, "location_id": 0, "started_at": t_work, "finished_at": t_work + h},
-    #     ]
-    #     sp = pd.DataFrame(list_dict)
-    #     sp.index.name = "id"
-    #     result = osna_method(sp)
-    #     sp["purpose"] = ["home", "work", "home", "work"]
-    #     assert_frame_equal(sp, result)
+    def test_multiple_users_with_only_one_location(self):
+        """Test that function can handle multiple users with only one location."""
+        t_leis = pd.Timestamp("2021-07-14 01:00:00", tz="utc")
+        t_work = pd.Timestamp("2021-07-14 18:00:00", tz="utc")
+        h = pd.Timedelta("1h")
+        list_dict = [
+            {"user_id": 0, "location_id": 0, "started_at": t_leis, "finished_at": t_leis + h},
+            {"user_id": 0, "location_id": 1, "started_at": t_work, "finished_at": t_work + h},
+            {"user_id": 1, "location_id": 0, "started_at": t_leis, "finished_at": t_leis + h},
+            {"user_id": 2, "location_id": 0, "started_at": t_work, "finished_at": t_work + h},
+        ]
+        sp = pd.DataFrame(list_dict)
+        sp.index.name = "id"
+        result = osna_method(sp)
+        sp["purpose"] = ["home", "work", "home", "work"]
+        assert_frame_equal(sp, result)
 
 
 class Test_osna_label_timeframes:
diff --git a/tests/analysis/test_tracking_quality.py b/tests/analysis/test_tracking_quality.py
index ff508a9..c22f42b 100644
--- a/tests/analysis/test_tracking_quality.py
+++ b/tests/analysis/test_tracking_quality.py
@@ -18,7 +18,7 @@ def testdata_sp_tpls_geolife_long():
 
     tpls["type"] = "tripleg"
     sp["type"] = "staypoint"
-    sp_tpls = sp.append(tpls, ignore_index=True).sort_values(by="started_at")
+    sp_tpls = pd.concat((sp, tpls), ignore_index=True).sort_values(by="started_at")
     return sp_tpls
 
 
diff --git a/tests/io/test_dataset_reader.py b/tests/io/test_dataset_reader.py
index 3e9f0f9..59b3d53 100644
--- a/tests/io/test_dataset_reader.py
+++ b/tests/io/test_dataset_reader.py
@@ -4,7 +4,6 @@ import os
 import pandas as pd
 import pytest
 from geopandas.testing import assert_geodataframe_equal
-from pandas.testing import assert_frame_equal
 from shapely.geometry import Point
 
 import trackintel as ti
@@ -171,21 +170,16 @@ class TestGeolife_add_modes_to_triplegs:
         triplegs, labels_raw = matching_data
 
         # add one record to the labels_raw: mode bus which are 1 min shorter than mode bike
-        labels_raw = (
-            labels_raw.reset_index()
-            .append(
-                [
-                    {
-                        "id": 2,
-                        # this record started 1 minute later than id=1 record - the final match ratio will be lower
-                        "started_at": labels_raw.iloc[-1]["started_at"] + datetime.timedelta(minutes=1),
-                        "finished_at": labels_raw.iloc[-1]["finished_at"],
-                        "mode": "bus",
-                    },
-                ]
-            )
-            .set_index("id")
-        )
+        labels_raw.reset_index(inplace=True)
+        entry = {
+            "id": 2,
+            # this record started 1 minute later than id=1 record - the final match ratio will be lower
+            "started_at": labels_raw.iloc[-1]["started_at"] + datetime.timedelta(minutes=1),
+            "finished_at": labels_raw.iloc[-1]["finished_at"],
+            "mode": "bus",
+        }
+        entry = pd.DataFrame([entry])
+        labels_raw = pd.concat((labels_raw, entry)).set_index("id")
         labels = {0: labels_raw}
 
         # the correct behaviour is to only choose one mode per tripleg id based on overlapping ratio
diff --git a/tests/io/test_from_geopandas.py b/tests/io/test_from_geopandas.py
index 3dc0edd..55a3e9a 100644
--- a/tests/io/test_from_geopandas.py
+++ b/tests/io/test_from_geopandas.py
@@ -98,7 +98,8 @@ class Test_Trackintel_Model:
             pd.Timestamp("2021-08-01 16:00:00", tz="Asia/Muscat"),
             pd.Timestamp("2021-08-01 16:00:00", tz="Pacific/Niue"),
         ]
-        pfs = _trackintel_model(example_positionfixes, tz_cols=["tracked_at"])
+        with pytest.warns(UserWarning, match="Assuming UTC timezone for column tracked_at"):
+            pfs = _trackintel_model(example_positionfixes, tz_cols=["tracked_at"])
         example_positionfixes["tracked_at"] = pd.to_datetime(example_positionfixes["tracked_at"], utc=True)
         assert_geodataframe_equal(pfs, example_positionfixes)
 
@@ -314,7 +315,8 @@ class TestRead_Trips_Gpd:
             pd.Timestamp("2021-08-01 16:00:00", tz="Asia/Muscat"),
             pd.Timestamp("2021-08-01 16:00:00", tz="Pacific/Niue"),
         ]
-        trips = read_trips_gpd(example_trips)
+        with pytest.warns(UserWarning, match="Assuming UTC timezone for column started_at"):
+            trips = read_trips_gpd(example_trips)
         example_trips["started_at"] = pd.to_datetime(example_trips["started_at"], utc=True)
         assert_geodataframe_equal(example_trips, trips)
 
diff --git a/tests/model/test_util.py b/tests/model/test_util.py
index 0b4a8bc..80f6b28 100644
--- a/tests/model/test_util.py
+++ b/tests/model/test_util.py
@@ -24,7 +24,8 @@ def example_triplegs():
 @pytest.fixture
 def load_positionfixes():
     """Load test positionfixes"""
-    pfs = ti.io.file.read_positionfixes_csv(os.path.join("tests", "data", "positionfixes.csv"), sep=";", index_col="id")
+    path = os.path.join("tests", "data", "positionfixes.csv")
+    pfs = ti.read_positionfixes_csv(path, sep=";", index_col="id", crs="EPSG:4326")
     # the correct speeds were computed manually in Python
     correct_speeds = np.array([8.82100607, 8.82100607, 0.36585538, 1.93127652, 19.60643425, 2.07086017]) / 3.6
     return pfs, correct_speeds
diff --git a/tests/preprocessing/test_positionfixes.py b/tests/preprocessing/test_positionfixes.py
index df9af75..aafe217 100644
--- a/tests/preprocessing/test_positionfixes.py
+++ b/tests/preprocessing/test_positionfixes.py
@@ -493,7 +493,7 @@ class TestGenerate_triplegs:
 
         sp = sp[["started_at", "finished_at", "user_id"]]
         tpls = tpls[["started_at", "finished_at", "user_id"]]
-        sp_tpls = sp.append(tpls)
+        sp_tpls = pd.concat((sp, tpls))
         sp_tpls.sort_values(by=["user_id", "started_at"], inplace=True)
 
         for user_id_this in sp["user_id"].unique():
diff --git a/tests/preprocessing/test_staypoints.py b/tests/preprocessing/test_staypoints.py
index 1cfab19..6abb57d 100644
--- a/tests/preprocessing/test_staypoints.py
+++ b/tests/preprocessing/test_staypoints.py
@@ -201,7 +201,7 @@ class TestGenerate_locations:
         )
 
         # create locations as grouped staypoints, another way to create locations
-        other_locs = pd.DataFrame(columns=["user_id", "id", "center"])
+        other_locs = []
         grouped_df = sp.groupby(["user_id", "location_id"])
         for combined_id, group in grouped_df:
             user_id, location_id = combined_id
@@ -214,9 +214,9 @@ class TestGenerate_locations:
 
                 # point geometry of place
                 temp_loc["center"] = Point(group.geometry.x.mean(), group.geometry.y.mean())
-                other_locs = other_locs.append(temp_loc, ignore_index=True)
+                other_locs.append(temp_loc)
 
-        other_locs = gpd.GeoDataFrame(other_locs, geometry="center", crs=sp.crs)
+        other_locs = gpd.GeoDataFrame(other_locs, columns=["user_id", "id", "center"], geometry="center", crs=sp.crs)
         other_locs.set_index("id", inplace=True)
 
         assert all(other_locs["center"] == locs["center"])
@@ -228,12 +228,12 @@ class TestGenerate_locations:
         sp = ti.read_staypoints_csv(sp_file, tz="utc", index_col="id")
         # take the first row and duplicate once
         sp = sp.head(1)
-        sp = sp.append(sp, ignore_index=True)
+        sp = pd.concat([sp, sp], ignore_index=True)
         # assign a different user_id to the second row
         sp.iloc[1, 4] = 1
 
         # duplicate for a certain number
-        sp = sp.append([sp] * 5, ignore_index=True)
+        sp = pd.concat([sp] * 6, ignore_index=True)
         _, locs_ds = sp.as_staypoints.generate_locations(
             method="dbscan", epsilon=10, num_samples=0, distance_metric="haversine", agg_level="dataset"
         )
diff --git a/tests/preprocessing/test_triplegs.py b/tests/preprocessing/test_triplegs.py
index 31ea27a..625eda1 100644
--- a/tests/preprocessing/test_triplegs.py
+++ b/tests/preprocessing/test_triplegs.py
@@ -284,22 +284,22 @@ class TestGenerate_trips:
         # test if generated staypoints/triplegs are equal (especially important for trip ids)
         assert_frame_equal(sp_tpls_loaded, sp_tpls, check_dtype=False)
 
-    # def test_generate_trips_id_management(self, example_triplegs_higher_gap_threshold):
-    #     """Test if we can generate the example trips based on example data."""
-    #     sp_tpls_loaded = pd.read_csv(os.path.join("tests", "data", "geolife_long", "sp_tpls.csv"), index_col="id")
-    #     sp_tpls_loaded["started_at"] = pd.to_datetime(sp_tpls_loaded["started_at"])
-    #     sp_tpls_loaded["started_at_next"] = pd.to_datetime(sp_tpls_loaded["started_at_next"])
-    #     sp_tpls_loaded["finished_at"] = pd.to_datetime(sp_tpls_loaded["finished_at"])
+    def test_generate_trips_id_management(self, example_triplegs_higher_gap_threshold):
+        """Test if we can generate the example trips based on example data."""
+        sp_tpls_loaded = pd.read_csv(os.path.join("tests", "data", "geolife_long", "sp_tpls.csv"), index_col="id")
+        sp_tpls_loaded["started_at"] = pd.to_datetime(sp_tpls_loaded["started_at"])
+        sp_tpls_loaded["started_at_next"] = pd.to_datetime(sp_tpls_loaded["started_at_next"])
+        sp_tpls_loaded["finished_at"] = pd.to_datetime(sp_tpls_loaded["finished_at"])
 
-    #     sp, tpls = example_triplegs_higher_gap_threshold
+        sp, tpls = example_triplegs_higher_gap_threshold
 
-    #     # generate trips and a joint staypoint/triplegs dataframe
-    #     gap_threshold = 15
-    #     sp, tpls, _ = generate_trips(sp, tpls, gap_threshold=gap_threshold)
-    #     sp_tpls = _create_debug_sp_tpls_data(sp, tpls, gap_threshold=gap_threshold)
+        # generate trips and a joint staypoint/triplegs dataframe
+        gap_threshold = 15
+        sp, tpls, _ = generate_trips(sp, tpls, gap_threshold=gap_threshold)
+        sp_tpls = _create_debug_sp_tpls_data(sp, tpls, gap_threshold=gap_threshold)
 
-    #     # test if generated staypoints/triplegs are equal (especially important for trip ids)
-    #     assert_frame_equal(sp_tpls_loaded, sp_tpls, check_dtype=False)
+        # test if generated staypoints/triplegs are equal (especially important for trip ids)
+        assert_frame_equal(sp_tpls_loaded, sp_tpls, check_dtype=False)
 
     def test_only_staypoints_in_trip(self):
         """Test that trips with only staypoints (non-activities) are deleted."""
@@ -372,9 +372,9 @@ def _create_debug_sp_tpls_data(sp, tpls, gap_threshold):
     # create table with relevant information from triplegs and staypoints.
     tpls["type"] = "tripleg"
     sp["type"] = "staypoint"
-    sp_tpls = sp[
-        ["started_at", "finished_at", "user_id", "type", "is_activity", "trip_id", "prev_trip_id", "next_trip_id"]
-    ].append(tpls[["started_at", "finished_at", "user_id", "type", "trip_id"]])
+    cols_sp = ["started_at", "finished_at", "user_id", "type", "is_activity", "trip_id", "prev_trip_id", "next_trip_id"]
+    cols_tpls = ["started_at", "finished_at", "user_id", "type", "trip_id"]
+    sp_tpls = pd.concat((sp[cols_sp], tpls[cols_tpls]))
 
     # transform nan to bool
     sp_tpls["is_activity"] = sp_tpls["is_activity"] == True
diff --git a/tests/preprocessing/test_util.py b/tests/preprocessing/test_util.py
index c99f838..d39c3e6 100644
--- a/tests/preprocessing/test_util.py
+++ b/tests/preprocessing/test_util.py
@@ -71,3 +71,15 @@ class TestExplodeAgg:
         returned_df = _explode_agg("id", "c", orig_df, agg_df)
         solution_df = pd.DataFrame(orig)
         assert_frame_equal(returned_df, solution_df)
+
+    def test_index_dtype_with_None(self):
+        """Test if dtype of index isn't changed with None values."""
+        orig = [
+            {"a": 1, "c": 0},
+        ]
+        agg = [{"id": [0, 1], "c": 0}, {"id": [], "c": 1}]
+        orig_df = pd.DataFrame(orig, columns=["a"])
+        agg_df = pd.DataFrame(agg)
+        returned_df = _explode_agg("id", "c", orig_df, agg_df)
+        solution_df = pd.DataFrame(orig)
+        assert_frame_equal(returned_df, solution_df)