Repository: IN-CORE/pyincore (language: python)

Issue:
pyincore util to convert population dislocation output for heatmap

pyincore util to convert population dislocation output for heatmap



Discussion / hints:

Fix patch:
diff --git a/pyincore/__init__.py b/pyincore/__init__.py
index 7ca599fef..161e3fe36 100644
--- a/pyincore/__init__.py
+++ b/pyincore/__init__.py
@@ -20,6 +20,8 @@
 from pyincore.restorationservice import RestorationService
 from pyincore.spaceservice import SpaceService
 from pyincore.utils.analysisutil import AnalysisUtil
+from pyincore.utils.popdisloutputprocessutil import PopdislOutputProcessUtil
+from pyincore.utils.cgeoutputprocess import CGEOutputProcess
 from pyincore.dataset import Dataset, InventoryDataset, DamageRatioDataset
 from pyincore.models.fragilitycurveset import FragilityCurveSet
 from pyincore.models.standardfragilitycurve import StandardFragilityCurve
diff --git a/pyincore/utils/popdisloutputprocessutil.py b/pyincore/utils/popdisloutputprocessutil.py
new file mode 100644
index 000000000..ad16ee7ee
--- /dev/null
+++ b/pyincore/utils/popdisloutputprocessutil.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2021 University of Illinois and others. All rights reserved.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License v2.0 which accompanies this distribution,
+# and is available at https://www.mozilla.org/en-US/MPL/2.0/
+
+import geopandas as gpd
+import pandas as pd
+from shapely import wkt
+
+
+class PopdislOutputProcessUtil:
+    """This class converts various population dislocation results outputs to json format and shapefiles."""
+
+    @staticmethod
+    def get_heatmap_shp(pop_dislocation_result, pop_dislocation_result_path=None,
+                        filter_on=True, filename="pop-disl-numprec.shp"):
+        """Convert and filter opulation dislocatin output to shapefile that contains only guid and numprec column
+            Args:
+                pop_dislocation_result (obj): IN-CORE output of populationdislocation.
+                pop_dislocation_result_path (string): Direct path of the csv of population dislocation result. A
+                fallback for the case that object of popdisl is not provided.
+                filename (str): Path and name to save json output file in. E.g "heatmap.shp"
+
+            Returns:
+                filename: full path and filename of the shapefile
+        """
+        if pop_dislocation_result_path:
+            df = pd.read_csv(pop_dislocation_result_path, low_memory=False)
+        else:
+            df = pop_dislocation_result.get_dataframe_from_csv()
+
+        df['geometry'] = df['geometry'].apply(wkt.loads)
+
+        # only keep dislocated & guid
+        if filter_on:
+            df = df[(df['dislocated']) & (df['guid'].notnull()) & (df["numprec"].notnull())]
+
+        # save as shapefile
+        gdf = gpd.GeoDataFrame(df, crs='epsg:4326')
+        gdf = gdf[["guid", "numprec", "geometry"]]
+        gdf.to_file(filename)
+
+        return filename


Test patch:
diff --git a/tests/pyincore/utils/joplin-pop-disl-numprec.cpg b/tests/pyincore/utils/joplin-pop-disl-numprec.cpg
new file mode 100644
index 000000000..cd89cb975
--- /dev/null
+++ b/tests/pyincore/utils/joplin-pop-disl-numprec.cpg
@@ -0,0 +1,1 @@
+ISO-8859-1
\ No newline at end of file
diff --git a/tests/pyincore/utils/joplin-pop-disl-numprec.dbf b/tests/pyincore/utils/joplin-pop-disl-numprec.dbf
new file mode 100644
index 000000000..d102b2d81
Binary files /dev/null and b/tests/pyincore/utils/joplin-pop-disl-numprec.dbf differ
diff --git a/tests/pyincore/utils/joplin-pop-disl-numprec.prj b/tests/pyincore/utils/joplin-pop-disl-numprec.prj
new file mode 100644
index 000000000..f45cbadf0
--- /dev/null
+++ b/tests/pyincore/utils/joplin-pop-disl-numprec.prj
@@ -0,0 +1,1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]
\ No newline at end of file
diff --git a/tests/pyincore/utils/joplin-pop-disl-numprec.shp b/tests/pyincore/utils/joplin-pop-disl-numprec.shp
new file mode 100644
index 000000000..cafba0e75
Binary files /dev/null and b/tests/pyincore/utils/joplin-pop-disl-numprec.shp differ
diff --git a/tests/pyincore/utils/joplin-pop-disl-numprec.shx b/tests/pyincore/utils/joplin-pop-disl-numprec.shx
new file mode 100644
index 000000000..fede93525
Binary files /dev/null and b/tests/pyincore/utils/joplin-pop-disl-numprec.shx differ
diff --git a/tests/pyincore/utils/test_popdisloutputprocessutil.py b/tests/pyincore/utils/test_popdisloutputprocessutil.py
new file mode 100644
index 000000000..0a94a5840
--- /dev/null
+++ b/tests/pyincore/utils/test_popdisloutputprocessutil.py
@@ -0,0 +1,67 @@
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License v2.0 which accompanies this distribution,
+# and is available at https://www.mozilla.org/en-US/MPL/2.0/
+
+from pyincore import IncoreClient, DataService, SpaceService
+from pyincore.analyses.populationdislocation import PopulationDislocation
+import pyincore.globals as pyglobals
+from pyincore import PopdislOutputProcessUtil
+
+
+def run_with_base_class():
+
+    # run population dislocation first
+    client = IncoreClient(pyglobals.INCORE_API_DEV_URL)
+
+    # Joplin population dislocation
+    building_dmg = "602d96e4b1db9c28aeeebdce"  # dev Joplin
+    housing_unit_alloc = "5df7c989425e0b00092c5eb4"  # dev Joplin
+    bg_data = "5df7cb0b425e0b00092c9464"  # Joplin 2ev2
+    value_loss = "602d508fb1db9c28aeedb2a5"
+    result_name = "joplin-pop-disl-results"
+    seed = 1111
+
+    pop_dis = PopulationDislocation(client)
+
+    pop_dis.load_remote_input_dataset("building_dmg", building_dmg)
+    pop_dis.load_remote_input_dataset("housing_unit_allocation", housing_unit_alloc)
+    pop_dis.load_remote_input_dataset("block_group_data", bg_data)
+    pop_dis.load_remote_input_dataset("value_poss_param", value_loss)
+
+    pop_dis.set_parameter("result_name", result_name)
+    pop_dis.set_parameter("seed", seed)
+
+    pop_dis.run_analysis()
+
+    #######################################################
+    # get result
+    pop_dislocation_result = pop_dis.get_output_dataset("result")
+    filename = PopdislOutputProcessUtil.get_heatmap_shp(pop_dislocation_result, filter_on=True,
+                                                        filename="joplin-pop-disl-numprec.shp")
+
+    # upload to incore services and put under commresilience space
+    datasvc = DataService(client)
+    dataset_prop = {
+        "title": "Joplin Population Dislocation For Heatmap Plotting",
+        "description": "only contain dislocated numprec for joplin playbook plotting usage",
+        "contributors": [],
+        "dataType": "incore:popdislocationShp",
+        "storedUrl": "",
+        "format": "shapefile"
+    }
+    response = datasvc.create_dataset(dataset_prop)
+    dataset_id = response['id']
+    files = ['joplin-pop-disl-numprec.shp',
+             'joplin-pop-disl-numprec.dbf',
+             'joplin-pop-disl-numprec.shx',
+             'joplin-pop-disl-numprec.prj']
+    datasvc.add_files_to_dataset(dataset_id, files)
+
+    # add to space
+    spacesvc = SpaceService(client)
+    spacesvc.add_dataset_to_space("5f99ba8b0ace240b22a82e00", dataset_id=dataset_id)  # commresilience
+    print(dataset_id + " successfully uploaded and move to commresilience space!")
+
+
+if __name__ == '__main__':
+    run_with_base_class()