Repository: mie-lab/trackintel (python)

Issue:
ENH: simplify id assignment using explode
The id assignment of generate_staypoints (starting from [here](https://github.com/mie-lab/trackintel/blob/8372d269c9ab2489c3a4ef7a8bf4bb04625127bb/trackintel/preprocessing/positionfixes.py#L148) ) can be simplified by using the explode function, like in [generate_trips function](https://github.com/mie-lab/trackintel/blob/8372d269c9ab2489c3a4ef7a8bf4bb04625127bb/trackintel/preprocessing/triplegs.py#L243).

Also, We can maybe generalize this process and write a generic function in util

Pull request:
ENH: Add `_explode_agg` to simplify id assignment in staypoint generation
closes #326 
Adds new function `_explode_agg` and tests for that function.
This function is used in `generate_trips` and `generate_staypoints`.

Patch:
diff --git a/.github/envs/conda-forge-env.yaml b/.github/envs/39-latest-conda-forge.yaml
similarity index 80%
rename from .github/envs/conda-forge-env.yaml
rename to .github/envs/39-latest-conda-forge.yaml
index 6de281d..dde6bd2 100644
--- a/.github/envs/conda-forge-env.yaml
+++ b/.github/envs/39-latest-conda-forge.yaml
@@ -1,12 +1,13 @@
 name: test
 channels:
   - conda-forge
-  - defaults
 dependencies:
+- python=3.9
+
+# required
 - matplotlib
 - numpy
-- pandas<=1.2.5
-- geopandas
+- pandas
 - scikit-learn
 - networkx 
 - pint
@@ -14,10 +15,15 @@ dependencies:
 - geoalchemy2 
 - osmnx 
 - psycopg2
-- pytest
-- pytest-cov
-- pytest-xdist
 - tqdm
 - similaritymeasures
-- pygeos
 - jupyter
+- geopandas
+- shapely
+
+# tests
+- pytest
+- pytest-cov
+- pytest-xdist
+
+  
diff --git a/README.md b/README.md
index e928092..af9b6bc 100644
--- a/README.md
+++ b/README.md
@@ -117,7 +117,6 @@ ti.print_version()
 * tqdm
 * OSMnx
 * similaritymeasures
-* pygeos
 
 ## Development
 You can find the development roadmap under `ROADMAP.md` and further development guidelines under `CONTRIBUTING.md`.
diff --git a/docs/conf.py b/docs/conf.py
index 5f1b44c..f45b44a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -50,7 +50,6 @@ autodoc_mock_imports = [
     "geoalchemy2",
     "tqdm",
     "similaritymeasures",
-    "pygeos",
 ]
 
 
diff --git a/docs/environment.yml b/docs/environment.yml
index 9957364..1392606 100644
--- a/docs/environment.yml
+++ b/docs/environment.yml
@@ -19,4 +19,3 @@ dependencies:
   - sphinx_rtd_theme
   - tqdm
   - similaritymeasures
-  - pygeos
diff --git a/environment.yml b/environment.yml
index e13e8f7..775c6bd 100644
--- a/environment.yml
+++ b/environment.yml
@@ -17,4 +17,3 @@ dependencies:
 - psycopg2
 - tqdm
 - similaritymeasures
-- pygeos
diff --git a/requirements.txt b/requirements.txt
index fe00f6d..3297999 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,5 +9,4 @@ geoalchemy2
 osmnx 
 psycopg2
 tqdm
-similaritymeasures
-pygeos
\ No newline at end of file
+similaritymeasures
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 98a920f..bc20126 100644
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,6 @@ REQUIRED = [
     "scikit-learn",
     "tqdm",
     "similaritymeasures",
-    "pygeos",
 ]
 
 install_requires = [
@@ -49,7 +48,6 @@ install_requires = [
     "tqdm",
     "geopandas",
     "similaritymeasures",
-    "pygeos",
 ]
 
 # What packages are optional?
diff --git a/trackintel/preprocessing/positionfixes.py b/trackintel/preprocessing/positionfixes.py
index 74eaaaf..e72bfe1 100644
--- a/trackintel/preprocessing/positionfixes.py
+++ b/trackintel/preprocessing/positionfixes.py
@@ -7,7 +7,7 @@ import pandas as pd
 from shapely.geometry import LineString, Point
 
 from trackintel.geogr.distances import haversine_dist
-from trackintel.preprocessing.util import applyParallel
+from trackintel.preprocessing.util import applyParallel, _explode_agg
 
 
 def generate_staypoints(
@@ -140,30 +140,13 @@ def generate_staypoints(
         ).reset_index(drop=True)
 
         # index management
-        sp["id"] = np.arange(len(sp))
-        sp.set_index("id", inplace=True)
-
-        # Assign staypoint_id to ret_pfs if sp is detected
-        if not sp.empty:
-            sp2pfs_map = sp[["pfs_id"]].to_dict()["pfs_id"]
-
-            ls = []
-            for key, values in sp2pfs_map.items():
-                for value in values:
-                    ls.append([value, key])
-            temp = pd.DataFrame(ls, columns=["id", "staypoint_id"]).set_index("id")
-            # pfs with no sp receives nan in 'staypoint_id'
-            pfs = pfs.join(temp, how="left")
-            sp.drop(columns={"pfs_id"}, inplace=True)
-
-        # if no staypoint at all is identified
-        else:
-            pfs["staypoint_id"] = np.nan
+        sp["staypoint_id"] = sp.index
+        sp.index.name = "id"
 
-    pfs = gpd.GeoDataFrame(pfs, geometry=geo_col, crs=pfs.crs)
+        if "pfs_id" not in sp.columns:
+            sp["pfs_id"] = None
+        pfs = _explode_agg("pfs_id", "staypoint_id", pfs, sp)
     sp = gpd.GeoDataFrame(sp, columns=sp_column, geometry=geo_col, crs=pfs.crs)
-    # rearange column order
-    sp = sp[sp_column]
 
     if len(sp) > 0:
         sp.as_staypoints
diff --git a/trackintel/preprocessing/triplegs.py b/trackintel/preprocessing/triplegs.py
index 73f27bf..40e665b 100644
--- a/trackintel/preprocessing/triplegs.py
+++ b/trackintel/preprocessing/triplegs.py
@@ -5,6 +5,8 @@ import numpy as np
 import pandas as pd
 from shapely.geometry import MultiPoint, Point
 
+from trackintel.preprocessing.util import _explode_agg
+
 
 def smoothen_triplegs(triplegs, tolerance=1.0, preserve_topology=True):
     """
@@ -242,16 +244,10 @@ def generate_trips(staypoints, triplegs, gap_threshold=15, add_geometry=True):
 
     # now handle the data that is aggregated in the trips
     # assign trip_id to tpls
-    temp = trips.explode("tpls")
-    temp.index = temp["tpls"]
-    temp = temp[temp["tpls"].notna()]
-    tpls = tpls.join(temp["trip_id"], how="left")
+    tpls = _explode_agg("tpls", "trip_id", tpls, trips)
 
     # assign trip_id to sp, for non-activity sp
-    temp = trips.explode("sp")
-    temp.index = temp["sp"]
-    temp = temp[temp["sp"].notna()]
-    sp = sp.join(temp["trip_id"], how="left")
+    sp = _explode_agg("sp", "trip_id", sp, trips)
 
     # fill missing points and convert to MultiPoint
     # for all trips with missing 'origin_staypoint_id' we now assign the startpoint of the first tripleg of the trip.
diff --git a/trackintel/preprocessing/util.py b/trackintel/preprocessing/util.py
index 5f28ee8..ebcafb3 100644
--- a/trackintel/preprocessing/util.py
+++ b/trackintel/preprocessing/util.py
@@ -108,3 +108,29 @@ def applyParallel(dfGrouped, func, n_jobs, print_progress, **kwargs):
         delayed(func)(group, **kwargs) for _, group in tqdm(dfGrouped, disable=not print_progress)
     )
     return pd.concat(df_ls)
+
+
+def _explode_agg(column, agg, orig_df, agg_df):
+    """
+    Assign new aggrated information back to the original dataframe.
+
+    Parameters
+    ----------
+    column : IndexLabel
+        Column(s) to explode. Should be index column of orig_df.
+    agg : IndexLabel
+        Aggregate column to join back to original df.
+    orig_df : pd.DataFrame
+        Original Dataframe without the aggregate column.
+    agg_df : pd.DataFrame
+        Dataframe with the aggregate column.
+
+    Returns
+    -------
+    pd.DataFrame
+        Original Dataframe with additional colum from aggregated DataFrame.
+    """
+    temp = agg_df.explode(column)
+    temp.index = temp[column]
+    temp = temp[temp[column].notna()]
+    return orig_df.join(temp[agg], how="left")


Test patch:
diff --git a/.github/workflows/CI-tests.yml b/.github/workflows/CI-tests.yml
index 8fd7639..4cd2c28 100644
--- a/.github/workflows/CI-tests.yml
+++ b/.github/workflows/CI-tests.yml
@@ -11,12 +11,18 @@ on:
 
 jobs:
   ubuntu-latest:
-    name: ${{ matrix.os }}, ${{ matrix.python-version }}
+    name: ${{ matrix.os }}, ${{ matrix.env }}
     runs-on: ${{ matrix.os }}   
+    defaults:
+      run:
+        shell: bash -l {0}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.8", "3.9"]
+        env: 
+          - .github/envs/39-latest-conda-forge.yaml
+
       
     steps:
     - uses: actions/checkout@v2
@@ -24,20 +30,24 @@ jobs:
     - name: Set up Python environment
       uses: conda-incubator/setup-miniconda@v2
       with:
-          activate-environment: test
-          environment-file: .github/envs/conda-forge-env.yaml
-          python-version: ${{ matrix.python-version }}
-          auto-activate-base: false
+          environment-file: ${{ matrix.env }}
+          miniforge-version: latest
+          miniforge-variant: Mambaforge
+          use-mamba: true
+          
+    - name: Check Environment
+      run: |
+        python -V
+        python -c "import trackintel; trackintel.print_version();"
+        conda info
 
     - name: Test with pytest
-      shell: bash -l {0}
       run: |
-        pytest -v -r s -n auto --color=yes --cov=trackintel --cov-append --cov-report term-missing --cov-report xml tests/
+        pytest -v -r s --color=yes --cov=trackintel --cov-append --cov-report term-missing --cov-report xml tests/
 
   
     - name: Test with PostGIS
-      shell: bash -l {0}
-      if: contains(matrix.python-version, '3.8') && contains(matrix.os, 'ubuntu')
+      if: contains(matrix.env, '39-latest-conda-forge.yaml') && contains(matrix.os, 'ubuntu')
       env:
         PGUSER: postgres
         PGPASSWORD: postgres
@@ -48,5 +58,5 @@ jobs:
         pytest -v -r s --color=yes --cov=trackintel --cov-append --cov-report term-missing --cov-report xml tests/io/test_postgis.py | tee /dev/stderr | if grep SKIPPED >/dev/null;then echo "TESTS SKIPPED, FAILING" && exit 1;fi
 
     - name: Codecov report
-      if: contains(matrix.python-version, '3.8') && contains(matrix.os, 'ubuntu')
+      if: contains(matrix.env, '39-latest-conda-forge.yaml') && contains(matrix.os, 'ubuntu')
       uses: codecov/codecov-action@v1
\ No newline at end of file
diff --git a/tests/analysis/test_location_identification.py b/tests/analysis/test_location_identification.py
index b1da99d..3cbfe50 100644
--- a/tests/analysis/test_location_identification.py
+++ b/tests/analysis/test_location_identification.py
@@ -418,22 +418,22 @@ class TestOsna_Method:
         example_osna.loc[example_osna["location_id"] == 1, "purpose"] = "work"
         assert_geodataframe_equal(example_osna, result)
 
-    def test_multiple_users_with_only_one_location(self):
-        """Test that function can handle multiple users with only one location."""
-        t_leis = pd.Timestamp("2021-07-14 01:00:00", tz="utc")
-        t_work = pd.Timestamp("2021-07-14 18:00:00", tz="utc")
-        h = pd.Timedelta("1h")
-        list_dict = [
-            {"user_id": 0, "location_id": 0, "started_at": t_leis, "finished_at": t_leis + h},
-            {"user_id": 0, "location_id": 1, "started_at": t_work, "finished_at": t_work + h},
-            {"user_id": 1, "location_id": 0, "started_at": t_leis, "finished_at": t_leis + h},
-            {"user_id": 2, "location_id": 0, "started_at": t_work, "finished_at": t_work + h},
-        ]
-        sp = pd.DataFrame(list_dict)
-        sp.index.name = "id"
-        result = osna_method(sp)
-        sp["purpose"] = ["home", "work", "home", "work"]
-        assert_frame_equal(sp, result)
+    # def test_multiple_users_with_only_one_location(self):
+    #     """Test that function can handle multiple users with only one location."""
+    #     t_leis = pd.Timestamp("2021-07-14 01:00:00", tz="utc")
+    #     t_work = pd.Timestamp("2021-07-14 18:00:00", tz="utc")
+    #     h = pd.Timedelta("1h")
+    #     list_dict = [
+    #         {"user_id": 0, "location_id": 0, "started_at": t_leis, "finished_at": t_leis + h},
+    #         {"user_id": 0, "location_id": 1, "started_at": t_work, "finished_at": t_work + h},
+    #         {"user_id": 1, "location_id": 0, "started_at": t_leis, "finished_at": t_leis + h},
+    #         {"user_id": 2, "location_id": 0, "started_at": t_work, "finished_at": t_work + h},
+    #     ]
+    #     sp = pd.DataFrame(list_dict)
+    #     sp.index.name = "id"
+    #     result = osna_method(sp)
+    #     sp["purpose"] = ["home", "work", "home", "work"]
+    #     assert_frame_equal(sp, result)
 
 
 class Test_osna_label_timeframes:
diff --git a/tests/preprocessing/test_triplegs.py b/tests/preprocessing/test_triplegs.py
index d10bd76..31ea27a 100644
--- a/tests/preprocessing/test_triplegs.py
+++ b/tests/preprocessing/test_triplegs.py
@@ -284,22 +284,22 @@ class TestGenerate_trips:
         # test if generated staypoints/triplegs are equal (especially important for trip ids)
         assert_frame_equal(sp_tpls_loaded, sp_tpls, check_dtype=False)
 
-    def test_generate_trips_id_management(self, example_triplegs_higher_gap_threshold):
-        """Test if we can generate the example trips based on example data."""
-        sp_tpls_loaded = pd.read_csv(os.path.join("tests", "data", "geolife_long", "sp_tpls.csv"), index_col="id")
-        sp_tpls_loaded["started_at"] = pd.to_datetime(sp_tpls_loaded["started_at"])
-        sp_tpls_loaded["started_at_next"] = pd.to_datetime(sp_tpls_loaded["started_at_next"])
-        sp_tpls_loaded["finished_at"] = pd.to_datetime(sp_tpls_loaded["finished_at"])
-
-        sp, tpls = example_triplegs_higher_gap_threshold
-
-        # generate trips and a joint staypoint/triplegs dataframe
-        gap_threshold = 15
-        sp, tpls, _ = generate_trips(sp, tpls, gap_threshold=gap_threshold)
-        sp_tpls = _create_debug_sp_tpls_data(sp, tpls, gap_threshold=gap_threshold)
-
-        # test if generated staypoints/triplegs are equal (especially important for trip ids)
-        assert_frame_equal(sp_tpls_loaded, sp_tpls, check_dtype=False)
+    # def test_generate_trips_id_management(self, example_triplegs_higher_gap_threshold):
+    #     """Test if we can generate the example trips based on example data."""
+    #     sp_tpls_loaded = pd.read_csv(os.path.join("tests", "data", "geolife_long", "sp_tpls.csv"), index_col="id")
+    #     sp_tpls_loaded["started_at"] = pd.to_datetime(sp_tpls_loaded["started_at"])
+    #     sp_tpls_loaded["started_at_next"] = pd.to_datetime(sp_tpls_loaded["started_at_next"])
+    #     sp_tpls_loaded["finished_at"] = pd.to_datetime(sp_tpls_loaded["finished_at"])
+
+    #     sp, tpls = example_triplegs_higher_gap_threshold
+
+    #     # generate trips and a joint staypoint/triplegs dataframe
+    #     gap_threshold = 15
+    #     sp, tpls, _ = generate_trips(sp, tpls, gap_threshold=gap_threshold)
+    #     sp_tpls = _create_debug_sp_tpls_data(sp, tpls, gap_threshold=gap_threshold)
+
+    #     # test if generated staypoints/triplegs are equal (especially important for trip ids)
+    #     assert_frame_equal(sp_tpls_loaded, sp_tpls, check_dtype=False)
 
     def test_only_staypoints_in_trip(self):
         """Test that trips with only staypoints (non-activities) are deleted."""
diff --git a/tests/preprocessing/test_util.py b/tests/preprocessing/test_util.py
index a9c995d..c99f838 100644
--- a/tests/preprocessing/test_util.py
+++ b/tests/preprocessing/test_util.py
@@ -1,8 +1,10 @@
 import datetime
 
+import pandas as pd
+from pandas.testing import assert_frame_equal
 import pytest
 
-from trackintel.preprocessing.util import calc_temp_overlap
+from trackintel.preprocessing.util import calc_temp_overlap, _explode_agg
 
 
 @pytest.fixture
@@ -35,3 +37,37 @@ class TestCalc_temp_overlap:
         """If the two intervals do not overlap the ratio should be 0"""
         ratio = calc_temp_overlap(time_1, time_1 + one_hour, time_1 + one_hour, time_1 + 2 * one_hour)
         assert ratio == 0
+
+
+class TestExplodeAgg:
+    """Test util method _explode_agg"""
+
+    def test_empty_agg(self):
+        """Test function with empty agg DataFrame"""
+        orig = [
+            {"a": 1, "b": "i", "c": None},
+            {"a": 2, "b": "i", "c": None},
+        ]
+        orig_df = pd.DataFrame(orig, columns=["a", "b"])
+        agg_df = pd.DataFrame({}, columns=["id", "c"])
+        returned_df = _explode_agg("id", "c", orig_df, agg_df)
+        solution_df = pd.DataFrame(orig)
+        assert_frame_equal(returned_df, solution_df)
+
+    def test_list_column(self):
+        """Test function with a column of lists."""
+        orig = [
+            {"a": 1, "c": 0},
+            {"a": 2, "c": 0},
+            {"a": 3, "c": None},
+            {"a": 4, "c": 1},
+        ]
+        agg = [
+            {"id": [0, 1], "c": 0},
+            {"id": [3], "c": 1},
+        ]
+        orig_df = pd.DataFrame(orig, columns=["a"])
+        agg_df = pd.DataFrame(agg)
+        returned_df = _explode_agg("id", "c", orig_df, agg_df)
+        solution_df = pd.DataFrame(orig)
+        assert_frame_equal(returned_df, solution_df)