added conversion tests

.github/workflows/docker-hub.yml

@@ -23,9 +23,9 @@ jobs:
     - name: Build Docker image
       run: |
         docker build \
-          -t joshuasundance/geospatial-data-converter:${{ github.ref_name }} \
-          -t joshuasundance/geospatial-data-converter:latest \
+          -t joshuasundance/geospatial_data_converter:${{ github.ref_name }} \
+          -t joshuasundance/geospatial_data_converter:latest \
           .
 
     - name: Push to Docker Hub
-      run: docker push -a joshuasundance/geospatial-data-converter
+      run: docker push -a joshuasundance/geospatial_data_converter

.github/workflows/hf-space.yml

@@ -18,4 +18,4 @@ jobs:
       env:
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |
-        git push https://joshuasundance:$HF_TOKEN@huggingface.co/spaces/joshuasundance/geospatial-data-converter main
+        git push https://joshuasundance:$HF_TOKEN@huggingface.co/spaces/joshuasundance/geospatial_data_converter main

Dockerfile

@@ -12,9 +12,9 @@ COPY ./requirements.txt /home/appuser/requirements.txt
 RUN pip install --user --no-cache-dir  --upgrade -r /home/appuser/requirements.txt
 
 COPY geospatial_data_converter/ /home/appuser/geospatial_data_converter/
+COPY tests/ /home/appuser/tests/
 
 WORKDIR /workspace
 EXPOSE 7860
 
 CMD ["streamlit", "run", "/home/appuser/geospatial_data_converter/app.py", "--server.port", "7860", "--server.address", "0.0.0.0", "--server.enableXsrfProtection=false"]
-#CMD ["/bin/bash"]

README.md

@@ -1,5 +1,5 @@
 ---
-title: geospatial-data-converter
+title: geospatial_data_converter
 emoji: 🌎
 colorFrom: green
 colorTo: blue
@@ -9,21 +9,21 @@ pinned: true
 tags: [geospatial, streamlit, docker]
 ---
 
-# geospatial-data-converter
+# geospatial_data_converter
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![python](https://img.shields.io/badge/Python-3.11-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org)
-![GitHub tag (with filter)](https://img.shields.io/github/v/tag/joshuasundance-swca/geospatial-data-converter)
+![GitHub tag (with filter)](https://img.shields.io/github/v/tag/joshuasundance-swca/geospatial_data_converter)
 
-[![Push to Docker Hub](https://github.com/joshuasundance-swca/geospatial-data-converter/actions/workflows/docker-hub.yml/badge.svg)](https://github.com/joshuasundance-swca/geospatial-data-converter/actions/workflows/docker-hub.yml)
-[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/joshuasundance/geospatial-data-converter/latest)](https://hub.docker.com/r/joshuasundance/geospatial-data-converter)
+[![Push to Docker Hub](https://github.com/joshuasundance-swca/geospatial_data_converter/actions/workflows/docker-hub.yml/badge.svg)](https://github.com/joshuasundance-swca/geospatial_data_converter/actions/workflows/docker-hub.yml)
+[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/joshuasundance/geospatial_data_converter/latest)](https://hub.docker.com/r/joshuasundance/geospatial_data_converter)
 
-[![Push to HuggingFace Space](https://github.com/joshuasundance-swca/geospatial-data-converter/actions/workflows/hf-space.yml/badge.svg)](https://github.com/joshuasundance-swca/geospatial-data-converter/actions/workflows/hf-space.yml)
-[![Open HuggingFace Space](https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg)](https://huggingface.co/spaces/joshuasundance/geospatial-data-converter)
+[![Push to HuggingFace Space](https://github.com/joshuasundance-swca/geospatial_data_converter/actions/workflows/hf-space.yml/badge.svg)](https://github.com/joshuasundance-swca/geospatial_data_converter/actions/workflows/hf-space.yml)
+[![Open HuggingFace Space](https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg)](https://huggingface.co/spaces/joshuasundance/geospatial_data_converter)
 
-![Code Climate maintainability](https://img.shields.io/codeclimate/maintainability/joshuasundance-swca/geospatial-data-converter)
-![Code Climate issues](https://img.shields.io/codeclimate/issues/joshuasundance-swca/geospatial-data-converter)
-![Code Climate technical debt](https://img.shields.io/codeclimate/tech-debt/joshuasundance-swca/geospatial-data-converter)
+![Code Climate maintainability](https://img.shields.io/codeclimate/maintainability/joshuasundance-swca/geospatial_data_converter)
+![Code Climate issues](https://img.shields.io/codeclimate/issues/joshuasundance-swca/geospatial_data_converter)
+![Code Climate technical debt](https://img.shields.io/codeclimate/tech-debt/joshuasundance-swca/geospatial_data_converter)
 
 [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff)
@@ -31,7 +31,7 @@ tags: [geospatial, streamlit, docker]
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 [![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
-![Known Vulnerabilities](https://snyk.io/test/github/joshuasundance-swca/geospatial-data-converter/badge.svg)
+![Known Vulnerabilities](https://snyk.io/test/github/joshuasundance-swca/geospatial_data_converter/badge.svg)
 
 This project showcases a simple geospatial data converter using [Streamlit](https://streamlit.io) and [GeoPandas](https://geopandas.org/).
 
@@ -45,11 +45,11 @@ This project showcases a simple geospatial data converter using [Streamlit](http
 - Download button for the converted data
 
 # Deployment
-`geospatial-data-converter` is deployed as a [Docker image](https://hub.docker.com/r/<your-dockerhub-username>/geospatial-data-converter) based on the `python:3.11-slim-bookworm` image.
+`geospatial_data_converter` is deployed as a [Docker image](https://hub.docker.com/r/<your-dockerhub-username>/geospatial_data_converter) based on the `python:3.11-slim-bookworm` image.
 
 ## With Docker (pull from Docker Hub)
 1. Run in terminal:
-`docker run -p 7860:7860 <your-dockerhub-username>/geospatial-data-converter:latest`
+`docker run -p 7860:7860 <your-dockerhub-username>/geospatial_data_converter:latest`
 2. Open http://localhost:8501 in your browser
 
 ## Docker Compose (build locally)
@@ -60,7 +60,7 @@ This project showcases a simple geospatial data converter using [Streamlit](http
 ## Kubernetes
 1. Clone the repo. Navigate to cloned repo directory
 2. Run bash script: `/bin/bash ./kubernetes/deploy.sh`
-3. Get the IP address for your new service: `kubectl get service geospatial-data-converter`
+3. Get the IP address for your new service: `kubectl get service geospatial_data_converter`
 
 # Links
 - [Streamlit](https://streamlit.io)

bumpver.toml

@@ -14,5 +14,5 @@ push = true
 "bumpver.toml" = [
     'current_version = "{version}"',
 ]
-"geospatial-data-converter/app.py" = ['__version__ = "{version}"']
-"kubernetes/resources.yaml" = ['        image: joshuasundance/geospatial-data-converter:{version}']
+"geospatial_data_converter/app.py" = ['__version__ = "{version}"']
+"kubernetes/resources.yaml" = ['        image: joshuasundance/geospatial_data_converter:{version}']

docker-compose.yml

@@ -1,8 +1,8 @@
 version: '3.8'
 
 services:
-  geospatial-data-converter:
-    image: geospatial-data-converter:latest
+  geospatial_data_converter:
+    image: geospatial_data_converter:latest
     build: .
     ports:
       - "${APP_PORT:-7860}:${APP_PORT:-7860}"
@@ -14,3 +14,8 @@ services:
       "--server.address", "0.0.0.0",
       "--server.enableXsrfProtection=false"
     ]
+  test:
+    image: geospatial_data_converter:latest  # Use the same image as your main service
+    build: .
+    working_dir: /home/appuser
+    command: ["/bin/bash"]

geospatial_data_converter/app.py

@@ -6,7 +6,10 @@ import streamlit as st
 from aiohttp import ClientSession
 from restgdf import FeatureLayer
 
-from utils import read_file, convert, output_format_dict
+try:
+    from utils import read_file, convert, output_format_dict
+except ImportError:
+    from .utils import read_file, convert, output_format_dict
 
 __version__ = "0.1.1"
 

geospatial_data_converter/kml_tricks.py

@@ -131,7 +131,7 @@ def extract_data_from_kml_code(kml_code: str) -> pd.DataFrame:
     """Extracts data from KML code into a DataFrame using SimpleData tags, excluding embedded tables in feature descriptions"""
 
     # Parse the KML source code
-    soup = bs4.BeautifulSoup(kml_code, "html.parser")
+    soup = bs4.BeautifulSoup(kml_code, features="xml")
 
     # Find all SchemaData tags (representing rows)
     schema_data_tags = soup.find_all("schemadata")

geospatial_data_converter/tests/test_kml.py

@@ -1,16 +0,0 @@
-import sys
-
-sys.path.append("..")
-from utils import convert, read_file
-
-
-def test_kml_coversion():
-    with open("boundary.kml", "rb") as f:
-        kml = read_file(f)
-    converted_data = convert(kml, "boundary_converted.kml", "kml")
-    with open("boundary_converted.kml", "wb") as f:
-        f.write(converted_data)
-
-
-def test_kmz_conversion():
-    pass

geospatial_data_converter/utils.py

@@ -5,7 +5,10 @@ from tempfile import TemporaryDirectory
 from typing import BinaryIO
 import geopandas as gpd
 
-from kml_tricks import load_ge_data
+try:
+    from kml_tricks import load_ge_data
+except ImportError:
+    from .kml_tricks import load_ge_data
 
 output_format_dict = {
     "ESRI Shapefile": ("shp", "zip", "application/zip"),  # must be zipped
@@ -22,7 +25,7 @@ def read_file(file: BinaryIO, *args, **kwargs) -> gpd.GeoDataFrame:
     ext = ext.lower().strip(".")
     if ext == "zip":
         with TemporaryDirectory() as tmp_dir:
-            tmp_file_path = os.path.join(tmp_dir, file.name)
+            tmp_file_path = os.path.join(tmp_dir, f"{basename}.{ext}")
             with open(tmp_file_path, "wb") as tmp_file:
                 tmp_file.write(file.read())
             return gpd.read_file(
@@ -33,7 +36,10 @@ def read_file(file: BinaryIO, *args, **kwargs) -> gpd.GeoDataFrame:
             )
     elif ext in ("kml", "kmz"):
         with TemporaryDirectory() as tmp_dir:
-            tmp_file_path = os.path.join(tmp_dir, file.name)
+            tmp_file_path = os.path.join(tmp_dir, f"{basename}.{ext}")
+            print(file.name)
+            print(basename)
+            print(tmp_file_path)
             with open(tmp_file_path, "wb") as tmp_file:
                 tmp_file.write(file.read())
             return load_ge_data(tmp_file_path)
@@ -60,7 +66,6 @@ def convert(gdf: gpd.GeoDataFrame, output_name: str, output_format: str) -> byte
     """Convert a GeoDataFrame to the specified format"""
     with TemporaryDirectory() as tmpdir:
         out_path = os.path.join(tmpdir, output_name)
-
         if output_format == "CSV":
             gdf.to_csv(out_path)
         else:

kubernetes/resources.yaml

@@ -1,20 +1,20 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: geospatial-data-converter-deployment
+  name: geospatial_data_converter-deployment
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: geospatial-data-converter
+      app: geospatial_data_converter
   template:
     metadata:
       labels:
-        app: geospatial-data-converter
+        app: geospatial_data_converter
     spec:
       containers:
-      - name: geospatial-data-converter
-        image: joshuasundance/geospatial-data-converter:0.1.1
+      - name: geospatial_data_converter
+        image: joshuasundance/geospatial_data_converter:0.1.1
         imagePullPolicy: Always
         resources:
           requests:
@@ -29,15 +29,15 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: geospatial-data-converter-service
+  name: geospatial_data_converter-service
   # configure on Azure and uncomment below to use a vnet
 #  annotations:
 #    service.beta.kubernetes.io/azure-load-balancer-internal: "true"
 #    service.beta.kubernetes.io/azure-load-balancer-ipv4: vnet.ip.goes.here
-#    service.beta.kubernetes.io/azure-dns-label-name: "geospatial-data-converter"
+#    service.beta.kubernetes.io/azure-dns-label-name: "geospatial_data_converter"
 spec:
   selector:
-    app: geospatial-data-converter
+    app: geospatial_data_converter
   ports:
     - protocol: TCP
       port: 80
@@ -47,11 +47,11 @@ spec:
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
-  name: geospatial-data-converter-network-policy
+  name: geospatial_data_converter-network-policy
 spec:
   podSelector:
     matchLabels:
-      app: geospatial-data-converter
+      app: geospatial_data_converter
   policyTypes:
   - Ingress
   ingress:

requirements.txt

@@ -3,5 +3,6 @@ beautifulsoup4==4.12.2
 geopandas==0.14.1
 lxml==4.9.3
 pyogrio==0.7.2
+pytest
 restgdf==0.9.7
 streamlit==1.28.2

tests/test_conversions.py

@@ -0,0 +1,26 @@
+import os
+import pytest
+from geospatial_data_converter.utils import convert, read_file
+
+input_exts = ["kml", "kmz", "geojson", "zip"]
+output_exts = ["KML", "ESRI Shapefile", "GeoJSON", "CSV", "OpenFileGDB"]
+output_format_dict = {
+    "ESRI Shapefile": "shp",
+    "GeoJSON": "geojson",
+    "CSV": "csv",
+    "KML": "kml",
+    "OpenFileGDB": "gdb",
+}
+
+
+@pytest.mark.parametrize("in_ext", input_exts)
+@pytest.mark.parametrize("out_ext", output_exts)
+def test_kml_coversion(in_ext: str, out_ext: str) -> None:
+    test_file = f"test.{in_ext}"
+    test_file_path = os.path.join(os.getcwd(), "tests", "test_data", test_file)
+    with open(test_file_path, "rb") as f:
+        kml = read_file(f)
+    out_file = f"test.{output_format_dict[out_ext]}"
+    converted_data = convert(kml, out_file, out_ext)
+    with open("test.kml", "wb") as f:
+        f.write(converted_data)