[refactor] use poetry also to build the aws docker images

Pipfile

@@ -1,31 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-sphinx = "*"
-sphinx-autodoc-typehints = "*"
-myst-parser = "*"
-
-[dev-packages]
-aws-lambda-powertools = "*"
-awslambdaric = "*"
-bson = "*"
-geopandas = "*"
-jmespath = "*"
-matplotlib = "*"
-numpy = "*"
-onnxruntime = "*"
-opencv-python-headless = "*"
-pillow = "*"
-pydantic = ">=2.0.3"
-python-dotenv = "*"
-rasterio = "*"
-requests = "*"
-pytest = "*"
-pytest-cov = "*"
-
-[requires]
-python_version = "3.11"
-python_full_version = "3.11.6"

dockerfiles/dockerfile-lambda-fastsam-api

@@ -3,10 +3,13 @@ FROM 686901913580.dkr.ecr.eu-west-1.amazonaws.com/lambda-gdal-runner:latest
 # Include global arg in this stage of the build
 ARG LAMBDA_TASK_ROOT="/var/task"
 ARG PYTHONPATH="${LAMBDA_TASK_ROOT}:${PYTHONPATH}:/usr/local/lib/python3/dist-packages"
+ENV VIRTUAL_ENV=${LAMBDA_TASK_ROOT}/.venv \
+    PATH="${LAMBDA_TASK_ROOT}/.venv/bin:$PATH"
 
 # Set working directory to function root directory
 WORKDIR ${LAMBDA_TASK_ROOT}
 COPY ./samgis ${LAMBDA_TASK_ROOT}/samgis
+COPY ./src ${LAMBDA_TASK_ROOT}/src
 COPY ./machine_learning_models ${LAMBDA_TASK_ROOT}/machine_learning_models
 
 RUN ls -l /usr/bin/which
@@ -19,16 +22,15 @@ RUN ls -l ${LAMBDA_TASK_ROOT}
 RUN ls -ld ${LAMBDA_TASK_ROOT}
 RUN ls -l ${LAMBDA_TASK_ROOT}/machine_learning_models
 RUN python -c "import sys; print(sys.path)"
-RUN python -c "import cv2"
 RUN python -c "import geopandas"
 RUN python -c "import onnxruntime"
 RUN python -c "import rasterio"
 RUN python -c "import awslambdaric"
-RUN python -m pip list
-RUN python -m pip freeze
+RUN python -c "import cv2"
 RUN df -h
 RUN ls -l /lambda-entrypoint.sh
 RUN ls -l ${LAMBDA_TASK_ROOT}/samgis/
+RUN ls -l ${LAMBDA_TASK_ROOT}/src/
 
 # ENTRYPOINT  ["/lambda-entrypoint.sh"]
-CMD [ "samgis.io.lambda_wrapper.lambda_handler" ]
+CMD [ "src.lambda_wrapper.lambda_handler" ]

dockerfiles/dockerfile-lambda-gdal-runner

@@ -1,28 +1,76 @@
-FROM python:3.11-slim-bookworm
-
 # Include global arg in this stage of the build
+ARG ARCH="x86_64"
 ARG LAMBDA_TASK_ROOT="/var/task"
 ARG PYTHONPATH="${LAMBDA_TASK_ROOT}:${PYTHONPATH}:/usr/local/lib/python3/dist-packages"
+ARG POETRY_NO_INTERACTION=1
+ARG POETRY_VIRTUALENVS_IN_PROJECT=1
+ARG POETRY_VIRTUALENVS_CREATE=1
+ARG POETRY_CACHE_DIR=/tmp/poetry_cache
 ARG RIE="https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie"
 
-RUN echo "ENV RIE: $RIE ..."
+
+FROM python:3.11-bookworm as builder
+
+ARG ARCH
+ARG LAMBDA_TASK_ROOT
+ARG PYTHONPATH
+ARG POETRY_NO_INTERACTION
+ARG POETRY_VIRTUALENVS_IN_PROJECT
+ARG POETRY_VIRTUALENVS_CREATE
+ARG POETRY_CACHE_DIR
+ARG RIE
+
+RUN echo "ARCH: $ARCH ..."
+
+RUN echo "ARG RIE: $RIE ..."
+RUN echo "ARG POETRY_CACHE_DIR: ${POETRY_CACHE_DIR} ..."
+RUN echo "ARG PYTHONPATH: $PYTHONPATH ..."
 
 # Set working directory to function root directory
 WORKDIR ${LAMBDA_TASK_ROOT}
-COPY requirements_pip.txt ${LAMBDA_TASK_ROOT}/requirements_pip.txt
-COPY requirements_dockerfile.txt ${LAMBDA_TASK_ROOT}/requirements_dockerfile.txt
+COPY requirements_poetry.txt pyproject.toml poetry.lock README.md ${LAMBDA_TASK_ROOT}/
 
 # avoid segment-geospatial exception caused by missing libGL.so.1 library
 RUN apt update && apt install -y libgl1 curl python3-pip && apt clean
-RUN ls -ld /usr/lib/*linux-gnu/libGL.so* || echo "libGL.so* not found..."
+RUN echo "check for libGL.so ..."
+RUN ls -ld /usr/lib/*-linux-gnu/libGL.so* || echo "libGL.so* not found/1..."
+RUN ls -ld /usr/lib/${ARCH}-linux-gnu/libGL.so* || echo "libGL.so* not found/2..."
 RUN which python
 RUN python --version
-RUN python -m pip install -r ${LAMBDA_TASK_ROOT}/requirements_pip.txt --upgrade --target ${LAMBDA_TASK_ROOT}
-RUN python -m pip install -r ${LAMBDA_TASK_ROOT}/requirements_dockerfile.txt --target ${LAMBDA_TASK_ROOT}
-RUN python -c "import sys;print(sys.path)"
-# RUN python -m pip install pillow awslambdaric aws-lambda-powertools httpx jmespath --target ${LAMBDA_TASK_ROOT}
+
+# poetry installation path is NOT within ${LAMBDA_TASK_ROOT}: not needed for runtime docker image
+RUN python -m pip install -r ${LAMBDA_TASK_ROOT}/requirements_poetry.txt
+
+RUN which poetry && poetry --version && poetry config --list
+RUN poetry config virtualenvs.path ${LAMBDA_TASK_ROOT}
+RUN echo "# poetry config --list #" && poetry config --list
+RUN poetry install --with aws_lambda --no-root
 
 RUN curl -Lo /usr/local/bin/aws-lambda-rie ${RIE}
+
+RUN echo "check for libGL.so"
+RUN ls -ld /usr/lib/*-linux-gnu/libGL.so* || echo "libGL.so* not found..."
+RUN ls -ld ${LAMBDA_TASK_ROOT}/.venv
+RUN ls -l ${LAMBDA_TASK_ROOT}/.venv
+
+
+FROM python:3.11-slim-bookworm as runtime
+
+ARG ARCH
+ARG LAMBDA_TASK_ROOT
+
+ENV VIRTUAL_ENV=${LAMBDA_TASK_ROOT}/.venv \
+    PATH="${LAMBDA_TASK_ROOT}/.venv/bin:$PATH"
+
+COPY --from=builder /usr/local/bin/aws-lambda-rie /usr/local/bin/aws-lambda-rie
+RUN echo "COPY --from=builder /usr/lib/${ARCH}-linux-gnu/libGL.so* /usr/lib/${ARCH}-linux-gnu/"
+COPY --from=builder /usr/lib/${ARCH}-linux-gnu/libGL.so* /usr/lib/${ARCH}-linux-gnu/
+COPY --from=builder ${LAMBDA_TASK_ROOT}/.venv ${LAMBDA_TASK_ROOT}/.venv
+
+RUN echo "new LAMBDA_TASK_ROOT after hidden venv copy => ${LAMBDA_TASK_ROOT}"
+RUN ls -ld ${LAMBDA_TASK_ROOT}/
+RUN ls -lA ${LAMBDA_TASK_ROOT}/
+
 RUN chmod +x /usr/local/bin/aws-lambda-rie
 
 COPY ./scripts/lambda-entrypoint.sh /lambda-entrypoint.sh

pyproject.toml

@@ -1,30 +1,50 @@
-[build-system]
-requires = ["hatchling", "hatch-requirements-txt"]
-build-backend = "hatchling.build"
-
-[project]
+[tool.poetry]
 name = "samgis"
 version = "1.0.0"
-authors = [
-  { name="Alessandro Trinca Tornidor", email="alessandro@trinca.tornidor.com" },
-]
 description = "A backend for machine learning instance segmentation on geospatial data even without dedicated graphics cards."
+authors = ["alessandro trinca tornidor <alessandro@trinca.tornidor.com>"]
+license = "MIT license"
 readme = "README.md"
-requires-python = ">=3.11"
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
-]
-dynamic = ["dependencies", "optional-dependencies"]
-
-[project.urls]
-Homepage = "https://github.com/trincadev/samgis-be"
-Issues = "https://github.com/trincadev/samgis-be/issues"
-
-[tool.hatch.metadata.hooks.requirements_txt]
-files = ["requirements_dockerfile.txt"]
-
-[tool.hatch.metadata.hooks.requirements_txt.optional-dependencies]
-dev = ["requirements_dev.txt"]
-docs = ["requirements.txt"]
+
+[tool.poetry.dependencies]
+bson = "^0.5.10"
+contextily = "^1.4.0"
+geopandas = "^0.14.1"
+numpy = "^1.26.2"
+onnxruntime = "^1.16.3"
+opencv-python-headless = "^4.8.1.78"
+python = "^3.11"
+python-dotenv = "^1.0.0"
+rasterio = "^1.3.9"
+requests = "^2.31.0"
+pillow = "^10.1.0"
+
+[tool.poetry.group.aws_lambda]
+optional = true
+
+[tool.poetry.group.aws_lambda.dependencies]
+aws-lambda-powertools = "^2.30.2"
+awslambdaric = "^2.0.8"
+jmespath = "^1.0.1"
+pydantic = ">=2.0.3"
+
+[tool.poetry.group.test]
+optional = true
+
+[tool.poetry.group.test.dependencies]
+pytest = "^7.4.3"
+pytest-cov = "^4.1.0"
+python-dotenv = "^1.0.0"
+
+[tool.poetry.group.docs]
+optional = true
+
+[tool.poetry.group.docs.dependencies]
+sphinx = "^7.2.6"
+sphinx-autodoc-typehints = "^1.25.2"
+sphinxcontrib-openapi = "^0.8.3"
+myst-parser = "^2.0.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements_pip.txt

@@ -1,3 +0,0 @@
-pip
-wheel
-setuptools

requirements_poetry.txt

@@ -0,0 +1 @@
+poetry==1.7.1

samgis/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

scripts/lambda-entrypoint.sh

@@ -1,7 +1,17 @@
 #!/bin/sh
 
+# default value for PYTHONBIN=/usr/local/bin/python
+# but it change because of poetry that it uses a virtualenv installed within the LAMBDA_TASK_ROOT=/var/task
+
+echo "python installation path and version:"
+which python
+python --version
+
+echo "lambda-entrypoint.sh: PYTHONPATH ${PYTHONPATH} ..."
+echo "lambda-entrypoint.sh: PATH ${PATH} ..."
+
 if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then
-  exec /usr/local/bin/aws-lambda-rie /usr/local/bin/python -m awslambdaric "$@"
+  exec /usr/local/bin/aws-lambda-rie python -m awslambdaric "$@"
 else
-  exec /usr/local/bin/python -m awslambdaric "$@"
+  exec python -m awslambdaric "$@"
 fi

tests/test_app.py

@@ -5,7 +5,8 @@ from unittest.mock import patch
 
 from awslambdaric.lambda_context import LambdaContext
 
-from samgis.io import lambda_helpers, lambda_wrapper
+from samgis.io import lambda_helpers
+from src import lambda_wrapper
 from tests.local_tiles_http_server import LocalTilesHttpServer
 
 
@@ -21,7 +22,7 @@ class TestAppFailures(unittest.TestCase):
             samexporter_predict_mocked,
             time_mocked
     ):
-        from samgis.io.lambda_wrapper import lambda_handler
+        from src.lambda_wrapper import lambda_handler
 
         time_mocked.return_value = 0
         get_parsed_request_body_mocked.value = {}
@@ -44,7 +45,7 @@ class TestAppFailures(unittest.TestCase):
     @patch.object(time, "time")
     @patch.object(lambda_wrapper, "get_parsed_request_body")
     def test_lambda_handler_400(self, get_parsed_request_body_mocked, time_mocked):
-        from samgis.io.lambda_wrapper import lambda_handler
+        from src.lambda_wrapper import lambda_handler
 
         time_mocked.return_value = 0
         get_parsed_request_body_mocked.return_value = {}
@@ -64,7 +65,7 @@ class TestAppFailures(unittest.TestCase):
 
     @patch.object(time, "time")
     def test_lambda_handler_422(self, time_mocked):
-        from samgis.io.lambda_wrapper import lambda_handler
+        from src.lambda_wrapper import lambda_handler
 
         time_mocked.return_value = 0
         event = {"body": {}, "version": 1.0}
@@ -95,7 +96,7 @@ class TestAppFailures(unittest.TestCase):
             samexporter_predict_mocked,
             time_mocked
     ):
-        from samgis.io.lambda_wrapper import lambda_handler
+        from src.lambda_wrapper import lambda_handler
         from tests import TEST_EVENTS_FOLDER
 
         time_mocked.return_value = 0
@@ -145,7 +146,7 @@ class TestAppFailures(unittest.TestCase):
         import xyzservices
         import shapely
 
-        from samgis.io.lambda_wrapper import lambda_handler
+        from src.lambda_wrapper import lambda_handler
         from tests import LOCAL_URL_TILE, TEST_EVENTS_FOLDER
 
         local_tile_provider = xyzservices.TileProvider(name="local_tile_provider", url=LOCAL_URL_TILE, attribution="")
@@ -190,7 +191,7 @@ class TestAppFailures(unittest.TestCase):
                 assert len(output_geojson.geoms) == expected_response_body["n_shapes_geojson"]
 
     def test_debug(self):
-        from samgis.io.lambda_wrapper import lambda_handler
+        from src.lambda_wrapper import lambda_handler
 
         input_event = {
             'bbox': {