Repository: swisstopo/swissgeol-boreholes-dataextraction

Issue:
API endpoint to access all bounding boxes contained in the page
[Migrated from LGVISIUM-103.]

This ticket is based on the discussion in Issue LGVISIUM-89. 

It would be good to give the end-user some more feedback on what is selected while he is dragging with the mouse, similar to how text is highlighted in the browser or in a PDF reader while a user is selecting text. In order to achieve this without latency, the Borehole application should request the bounding boxes for all words on a particular page of the document so this data is already available on the client side when the user starts selecting and can then be used to add word-by-word highlighting onto the PNG image while the user is selecting.

Related issue in the boreholes.swissgeol.ch repo: https://github.com/swisstopo/swissgeol-boreholes-suite/issues/1638

Discussion:

Solution patch:
diff --git a/src/app/api/v1/endpoints/bounding_boxes.py b/src/app/api/v1/endpoints/bounding_boxes.py
new file mode 100644
index 00000000..aa05139d
--- /dev/null
+++ b/src/app/api/v1/endpoints/bounding_boxes.py
@@ -0,0 +1,44 @@
+"""This module defines the FastAPI endpoint for extracting information from a PDF borehole document."""
+
+from pathlib import Path
+
+from app.common.helpers import load_pdf_page, load_png
+from app.common.schemas import BoundingBox, BoundingBoxesResponse
+from stratigraphy.text.extract_text import extract_text_lines
+
+
+def bounding_boxes(filename: Path, page_number: int) -> BoundingBoxesResponse:
+    """Return the bounding boxes of all words on a PDF page.
+
+    Args:
+        filename (Path): The filename of the PDF to extract bounding boxes from.
+        page_number (int): The number (1-based) of the PDF page to extract bounding boxes from.
+
+    Returns:
+        BoundingBoxesResponse: The response containing a list of bounding boxes for all words on the page.
+    """
+    pdf_page = load_pdf_page(filename, page_number)
+    pdf_page_width = pdf_page.rect.width
+    pdf_page_height = pdf_page.rect.height
+
+    # Load the PNG file from AWS
+    png_page = load_png(filename, page_number)
+    png_page_width = png_page.shape[1]
+    png_page_height = png_page.shape[0]
+
+    # Extract the text
+    text_lines = extract_text_lines(pdf_page)
+
+    bboxes = [
+        # Convert the bounding box to PNG coordinates
+        BoundingBox.load_from_fitz_rect(word.rect).rescale(
+            original_height=pdf_page_height,
+            original_width=pdf_page_width,
+            target_height=png_page_height,
+            target_width=png_page_width,
+        )
+        for line in text_lines
+        for word in line.words
+    ]
+
+    return BoundingBoxesResponse(bounding_boxes=bboxes)
diff --git a/src/app/api/v1/endpoints/extract_data.py b/src/app/api/v1/endpoints/extract_data.py
index 84fc294d..467c9702 100644
--- a/src/app/api/v1/endpoints/extract_data.py
+++ b/src/app/api/v1/endpoints/extract_data.py
@@ -1,10 +1,9 @@
 """This module defines the FastAPI endpoint for extracting information from PDF borehole document."""
 
 import re
-from pathlib import Path
 
 import fitz
-from app.common.aws import load_pdf_from_aws, load_png_from_aws
+from app.common.helpers import load_pdf_page, load_png
 from app.common.schemas import (
     BoundingBox,
     Coordinates,
@@ -16,8 +15,9 @@
     FormatTypes,
 )
 from fastapi import HTTPException
+from stratigraphy.lines.line import TextLine
 from stratigraphy.metadata.coordinate_extraction import CoordinateExtractor, LV03Coordinate, LV95Coordinate
-from stratigraphy.text.extract_text import extract_text_lines_from_bbox
+from stratigraphy.text.extract_text import extract_text_lines
 
 
 def extract_data(extract_data_request: ExtractDataRequest) -> ExtractDataResponse:
@@ -35,22 +35,11 @@ def extract_data(extract_data_request: ExtractDataRequest) -> ExtractDataRespons
         ExtractDataResponse: The extracted information with the bounding box where the information was found.
         In PNG coordinates.
     """
-    # Load the PNG image
-    pdf_document = load_pdf_from_aws(extract_data_request.filename)
-
-    # Load the page from the PDF document
-    pdf_page = pdf_document.load_page(extract_data_request.page_number - 1)
+    pdf_page = load_pdf_page(extract_data_request.filename, extract_data_request.page_number)
     pdf_page_width = pdf_page.rect.width
     pdf_page_height = pdf_page.rect.height
 
-    # Load the PNG image the boreholes app is showing to the user
-    # Convert the PDF filename to a PNG filename: "10012.pdf" -> 'dataextraction/10012-1.png'
-    # Remove the file extension and replace it with '.png'
-    base_filename = extract_data_request.filename.stem
-    png_filename = Path(f"{base_filename}-{extract_data_request.page_number}.png")
-
-    # Load the PNG file from AWS
-    png_page = load_png_from_aws(png_filename)
+    png_page = load_png(extract_data_request.filename, extract_data_request.page_number)
     png_page_width = png_page.shape[1]
     png_page_height = png_page.shape[0]
 
@@ -62,15 +51,23 @@ def extract_data(extract_data_request: ExtractDataRequest) -> ExtractDataRespons
         target_width=pdf_page_width,
     )  # bbox in PDF coordinates
 
-    # If the PDF page is rotated, rotate the bounding box
-    user_defined_rect_with_rotation = user_defined_bbox.to_fitz_rect() * pdf_page.derotation_matrix
+    # Select words whose middle-point is in the user-defined bbox
+    text_lines = []
+    for text_line in extract_text_lines(pdf_page):
+        words = [
+            word
+            for word in text_line.words
+            if user_defined_bbox.to_fitz_rect().contains(
+                fitz.Point((word.rect.x0 + word.rect.x1) / 2, (word.rect.y0 + word.rect.y1) / 2)
+            )
+        ]
+        if words:
+            text_lines.append(TextLine(words))
 
     # Extract the information based on the format type
     if extract_data_request.format == FormatTypes.COORDINATES:
         # Extract the coordinates and bounding box
-        extracted_coords: ExtractCoordinatesResponse | None = extract_coordinates(
-            extract_data_request, pdf_page, user_defined_rect_with_rotation
-        )
+        extracted_coords: ExtractCoordinatesResponse | None = extract_coordinates(extract_data_request, text_lines)
 
         # Convert the bounding box to PNG coordinates and return the response
         return ExtractCoordinatesResponse(
@@ -84,7 +81,7 @@ def extract_data(extract_data_request: ExtractDataRequest) -> ExtractDataRespons
         )
 
     elif extract_data_request.format == FormatTypes.TEXT:
-        extracted_text: ExtractTextResponse = extract_text(pdf_page, user_defined_rect_with_rotation)
+        extracted_text: ExtractTextResponse = extract_text(text_lines)
 
         # Convert the bounding box to PNG coordinates and return the response
         return ExtractTextResponse(
@@ -97,7 +94,7 @@ def extract_data(extract_data_request: ExtractDataRequest) -> ExtractDataRespons
             text=extracted_text.text,
         )
     elif extract_data_request.format == FormatTypes.NUMBER:
-        extracted_number = extract_number(pdf_page, user_defined_rect_with_rotation)
+        extracted_number = extract_number(text_lines)
 
         # Convert the bounding box to PNG coordinates and return the response
         return ExtractNumberResponse(
@@ -113,18 +110,14 @@ def extract_data(extract_data_request: ExtractDataRequest) -> ExtractDataRespons
         raise ValueError("Invalid format type.")
 
 
-def extract_coordinates(
-    extract_data_request: ExtractDataRequest, pdf_page: fitz.Page, user_defined_bbox: fitz.Rect
-) -> ExtractDataResponse:
-    """Extract coordinates from a PDF document.
+def extract_coordinates(extract_data_request: ExtractDataRequest, text_lines: list[TextLine]) -> ExtractDataResponse:
+    """Extract coordinates from a collection of text lines.
 
-    The coordinates are extracted from the user-defined bounding box. The coordinates are extracted in the
-    Swiss coordinate system (LV03 or LV95). The user_defined_bbox is in PDF coordinates.
+    The coordinates are extracted in the Swiss coordinate system (LV03 or LV95).
 
     Args:
         extract_data_request (ExtractDataRequest): The request data. The page number is 1-based.
-        pdf_page (fitz.Page): The PDF page.
-        user_defined_bbox (fitz.Rect): The user-defined bounding box. The bounding box is in PDF coordinates.
+        text_lines (list[TextLine]): The text lines to extract the coordinates from.
 
     Returns:
         ExtractDataResponse: The extracted coordinates. The coordinates are in the Swiss coordinate
@@ -148,9 +141,7 @@ def create_response(coord, srs):
         )
 
     coord_extractor = CoordinateExtractor()
-    extracted_coord = coord_extractor.extract_coordinates_from_bbox(
-        pdf_page, extract_data_request.page_number, user_defined_bbox
-    )
+    extracted_coord = coord_extractor.extract_coordinates_aggregated(text_lines, extract_data_request.page_number)
 
     if isinstance(extracted_coord, LV03Coordinate):
         return create_response(extracted_coord, "LV03")
@@ -161,21 +152,16 @@ def create_response(coord, srs):
     raise HTTPException(status_code=404, detail="Coordinates not found.")
 
 
-def extract_text(pdf_page: fitz.Page, user_defined_bbox: fitz.Rect) -> ExtractDataResponse:
-    """Extract text from a PDF Document. The text is extracted from the user-defined bounding box.
+def extract_text(text_lines: list[TextLine]) -> ExtractTextResponse:
+    """Extract text from a collection of text lines.
 
     Args:
-        pdf_page (fitz.Page): The PDF page.
-        user_defined_bbox (fitz.Rect): The user-defined bounding box. The bounding box is in PDF coordinates.
+        text_lines (list[TextLine]): The text lines to extract the numbers from.
 
     Returns:
-        ExtractDataResponse: The extracted text.
+        ExtractTextResponse: The extracted text.
     """
-    # Extract the text
-    text_lines = extract_text_lines_from_bbox(pdf_page, user_defined_bbox)
-
-    # Convert the text lines to a string
-    text = " ".join([text_line.text for text_line in text_lines])
+    text = " ".join([line.text for line in text_lines])
 
     text_based_bbox = fitz.Rect()
     for text_line in text_lines:
@@ -188,20 +174,15 @@ def extract_text(pdf_page: fitz.Page, user_defined_bbox: fitz.Rect) -> ExtractDa
         raise HTTPException(status_code=404, detail="Text not found.")
 
 
-def extract_number(pdf_page: fitz.Page, user_defined_bbox: fitz.Rect) -> ExtractNumberResponse:
-    """Extract numbers from a PDF document. The numbers are extracted from the user-defined bounding box.
+def extract_number(text_lines: list[TextLine]) -> ExtractNumberResponse:
+    """Extract numbers from a collection of text lines.
 
     Args:
-        pdf_page (fitz.Page): The PDF page.
-        user_defined_bbox (fitz.Rect): The user-defined bounding box. The bounding box is in PDF coordinates.
+        text_lines (list[TextLine]): The text lines to extract the numbers from.
 
     Returns:
-        ExtractDataResponse: The extracted number with bbox.
+        ExtractNumberResponse: The extracted number with bbox.
     """
-    # Extract the text
-    text_lines = extract_text_lines_from_bbox(pdf_page, user_defined_bbox)
-
-    # Extract the number
     for text_line in text_lines:
         number = extract_number_from_text(text_line.text)
         if number:
diff --git a/src/app/api/v1/router.py b/src/app/api/v1/router.py
index a77a4495..9267ee84 100644
--- a/src/app/api/v1/router.py
+++ b/src/app/api/v1/router.py
@@ -1,8 +1,11 @@
 """Main router for the app."""
 
+from app.api.v1.endpoints.bounding_boxes import bounding_boxes
 from app.api.v1.endpoints.create_pngs import create_pngs
 from app.api.v1.endpoints.extract_data import extract_data
 from app.common.schemas import (
+    BoundingBoxesRequest,
+    BoundingBoxesResponse,
     ExtractCoordinatesResponse,
     ExtractDataRequest,
     ExtractNumberResponse,
@@ -65,6 +68,46 @@ def post_create_pngs(request: PNGRequest) -> PNGResponse:
     return create_pngs(request.filename)
 
 
+####################################################################################################
+### Bounding Boxes for Words
+####################################################################################################
+@router.post(
+    "/bounding_boxes",
+    tags=["bounding_boxes"],
+    responses={
+        400: {"model": BadRequestResponse, "description": "Bad request"},
+        404: {
+            "model": BadRequestResponse,
+            "description": "Failed to load PDF document. The filename is not found in the bucket.",
+        },
+        500: {"model": BadRequestResponse, "description": "Internal server error"},
+    },
+)
+def get_bounding_boxes(request: BoundingBoxesRequest) -> BoundingBoxesResponse:
+    """Obtain bounding boxes (in PNG pixel coordinates) for all words that are found on the requested PDF page.
+
+    ### Prerequisites
+    The text in the PDF must be digitally readable, i.e. either a digitally-born PDF, or a PDF where OCR has already
+    been executed.
+
+    Ensure that the PDF file has been processed by the create_pngs endpoint first.
+
+    ### Request Body
+    - **request** (`BoundingBoxesRequest`): Contains the `filename` of the PDF document in the S3 bucket and the page
+    number.
+
+    ### Returns
+    - **BoundingBoxesResponse**: Response containing a list of bounding boxes.
+
+    ### Status Codes
+    - **200 OK**: The bounding were successfully found.
+    - **400 Bad Request**: The request format or content is invalid. Verify that `filename` is correctly specified.
+    - **404 Not Found**: PDF file not found in S3 bucket.
+    - **500 Internal Server Error**: An error occurred on the server while obtaining the bounding boxes.
+    """
+    return bounding_boxes(request.filename, request.page_number)
+
+
 ####################################################################################################
 ### Extract Data
 ####################################################################################################
@@ -83,10 +126,8 @@ def post_extract_data(
 ) -> ExtractCoordinatesResponse | ExtractTextResponse | ExtractNumberResponse:
     """Extract specified data from a given document based on the bounding box coordinates and format.
 
-    Behavior of the data extraction from the specified bounding box is the following: extraction on a per-letter
-    basis, which means that as soon as the specified bounding box overlaps (partially or fully) with a letter
-    or number, then this character is added to the extracted text. This behavior is consistent with the
-    clipping behavior of the `PyMuPDF` library.
+    Text is extracted on a word-by-word basis, whereby a word is included if its center point is within the bounding
+    box that is provided by the user in the request.
 
     ### Prerequisites
     Ensure that the PDF file has been processed by the create_pngs endpoint first.
diff --git a/src/app/common/helpers.py b/src/app/common/helpers.py
index 16366146..f141c836 100644
--- a/src/app/common/helpers.py
+++ b/src/app/common/helpers.py
@@ -1 +1,38 @@
 """Helper functions for the app."""
+
+from pathlib import Path
+
+import numpy as np
+import pymupdf
+from app.common.aws import load_pdf_from_aws, load_png_from_aws
+
+
+def load_pdf_page(filename: Path, page_number: int) -> pymupdf.Page:
+    """Loads the page from the PDF document.
+
+    Args:
+        filename: name of the PDF file to be loaded from S3.
+        page_number: number of the page (1-based)
+
+    Returns: the PDF page as a PyMuPDF Page object
+    """
+    pdf_document = load_pdf_from_aws(filename)
+    return pdf_document.load_page(page_number - 1)
+
+
+def load_png(filename: Path, page_number: int) -> np.ndarray:
+    """Loads the PNG image representation of a PDF page.
+
+    Convert the PDF filename to a PNG filename: "10012.pdf" -> 'dataextraction/10012-1.png'
+
+    Args:
+        filename: name of the corresponding PDF file on S3
+        page_number: number of the page (1-based)
+
+    Returns: the image as a numpy array
+    """
+    base_filename = filename.stem
+    png_filename = Path(f"{base_filename}-{page_number}.png")
+
+    # Load the PNG file from AWS
+    return load_png_from_aws(png_filename)
diff --git a/src/app/common/schemas.py b/src/app/common/schemas.py
index d6c5e830..96654619 100644
--- a/src/app/common/schemas.py
+++ b/src/app/common/schemas.py
@@ -194,6 +194,70 @@ class Coordinates(BaseModel):
     )
 
 
+class BoundingBoxesRequest(ABC, BaseModel):
+    """Request schema for the `bounding_boxes` endpoint.
+
+    ### Fields
+    Each field below includes inline examples to aid users in creating requests. See `json_schema_extra`
+    for a complete example.
+
+    **Attributes:**
+    - **filename** (`Path`): Path to the PDF file. _Example_: `"document.pdf"`
+    - **page_number** (`int`): Target page for data extraction. This is a 1-based index. _Example_: `1`
+
+    ### Validation
+    Custom validators ensure data integrity:
+    - **Filename Validator:** Ensures filename is not empty.
+    - **Page Number Validator:** Confirms page number is positive.
+    """
+
+    filename: Path = Field(
+        ...,
+        description="""Path to the input PDF document file that contains the data to be extracted. This should be
+        a valid file path, and the file should be accessible to the API.""",
+        example=Path("document.pdf"),
+    )
+    page_number: int = Field(
+        ...,
+        description="""Page number within the document where the extraction is to be performed. This is a 1-based 
+        index (e.g., 1 for the first page), applicable for multi-page files like PDFs.""",
+        example=1,
+    )
+
+    @field_validator("filename", mode="before")
+    @classmethod
+    def validate_filename(cls, value: str) -> str:
+        """Ensure the filename is not empty."""
+        return validate_filename(value)
+
+    @field_validator("page_number")
+    @classmethod
+    def page_number_must_be_positive(cls, v: int) -> int:
+        """Validate that the page number is positive."""
+        if v <= 0:
+            raise ValueError("Page number must be a positive integer")
+        return v
+
+    class Config:
+        """Make it possible to define an example for the entire request model in the Swagger UI.
+
+        The schema_extra attribute inside the Config class allows you to define a complete
+        example for the entire request model.
+        """
+
+        json_schema_extra = {"example": {"filename": "10012.pdf", "page_number": 1}}
+
+
+class BoundingBoxesResponse(BaseModel):
+    """Response schema for the `bounding_boxes` endpoint, representing the bounding boxes of words on the page."""
+
+    bounding_boxes: list[BoundingBox] = Field(
+        ...,
+        description="""List of bounding boxes for all words that are found on the requested page.""",
+        example=[{"x0": 0.0, "y0": 0.0, "x1": 100.0, "y1": 100.0}, {"x0": 150.0, "y0": 20.0, "x1": 220.0, "y1": 40.0}],
+    )
+
+
 class ExtractDataRequest(ABC, BaseModel):
     """Request schema for the `extract_data` endpoint.
 
diff --git a/src/stratigraphy/metadata/coordinate_extraction.py b/src/stratigraphy/metadata/coordinate_extraction.py
index 6e919a73..2184aa92 100644
--- a/src/stratigraphy/metadata/coordinate_extraction.py
+++ b/src/stratigraphy/metadata/coordinate_extraction.py
@@ -9,7 +9,7 @@
 import regex
 from stratigraphy.data_extractor.data_extractor import DataExtractor, ExtractedFeature
 from stratigraphy.lines.line import TextLine
-from stratigraphy.text.extract_text import extract_text_lines_from_bbox
+from stratigraphy.text.extract_text import extract_text_lines
 
 logger = logging.getLogger(__name__)
 
@@ -282,9 +282,7 @@ def _match_text_with_rect(
             results.append((match, rect))
         return results
 
-    def extract_coordinates_from_bbox(
-        self, page: fitz.Page, page_number: int, bbox: fitz.Rect | None = None
-    ) -> Coordinate | None:
+    def extract_coordinates_aggregated(self, text_lines: list[TextLine], page_number: int) -> Coordinate | None:
         """Extracts the coordinates from a borehole profile.
 
         Processes the borehole profile page by page and tries to find the coordinates in the respective text of the
@@ -298,12 +296,10 @@ def extract_coordinates_from_bbox(
         Returns:
             Coordinate | None: the extracted coordinates (if any)
         """
-        lines = extract_text_lines_from_bbox(page, bbox)
-
         found_coordinates = (
-            self.get_coordinates_with_x_y_labels(lines, page_number)
-            or self.get_coordinates_near_key(lines, page_number)
-            or self.get_coordinates_from_lines(lines, page_number)
+            self.get_coordinates_with_x_y_labels(text_lines, page_number)
+            or self.get_coordinates_near_key(text_lines, page_number)
+            or self.get_coordinates_from_lines(text_lines, page_number)
         )
 
         if len(found_coordinates) > 0:
@@ -329,6 +325,6 @@ def extract_coordinates(self, document: fitz.Document) -> Coordinate | None:
             Coordinate | None: the extracted coordinates (if any)
         """
         for page in document:
+            text_lines = extract_text_lines(page)
             page_number = page.number + 1  # page.number is 0-based
-
-            return self.extract_coordinates_from_bbox(page, page_number)
+            return self.extract_coordinates_aggregated(text_lines, page_number)
diff --git a/src/stratigraphy/text/extract_text.py b/src/stratigraphy/text/extract_text.py
index bb58d41b..69a9cf39 100644
--- a/src/stratigraphy/text/extract_text.py
+++ b/src/stratigraphy/text/extract_text.py
@@ -11,7 +11,6 @@ def extract_text_lines(page: fitz.Page) -> list[TextLine]:
 
     Args:
         page (fitz.page): the page to extract text from
-        page_number (int): the page number (first page is 1)
 
     Returns:
         list[TextLine]: A list of text lines.


Test patch:
diff --git a/tests/test_data_extraction_from_bbox.py b/tests/test_data_extraction_from_bbox.py
index 1b0698b0..401b2240 100644
--- a/tests/test_data_extraction_from_bbox.py
+++ b/tests/test_data_extraction_from_bbox.py
@@ -119,13 +119,13 @@ def test_extract_text_success(test_client: TestClient, upload_test_pdf, upload_t
         "Bohrmeister : Dragnic "
         "Ausführungsdatum 2.-3. 9. 1995 "
         "Koordinaten : 615 790 / 157 500 "
-        "Kote Bezugspunkt: ~788,6 m ü. M."
+        "Kote Bezugspunkt : ~788,6 m ü. M."
     )
 
     request = ExtractDataRequest(
         filename=TEST_PDF_KEY.name,
         page_number=1,
-        bbox=BoundingBox(x0=0, y0=0, x1=1000, y1=1000),
+        bbox=BoundingBox(x0=0, y0=0, x1=1020, y1=1000),
         format=FormatTypes.TEXT,
     )
     response = test_client.post("/api/V1/extract_data", content=request.model_dump_json())
@@ -220,10 +220,8 @@ def test_clipping_behavior(test_client: TestClient, upload_test_pdf, upload_test
     assert json_response["text"] == target_text
 
     ####################################################################################################
-    ### Extract Data on Normal PDF with bounding box with only one part of one word selected out of the text.
+    ### Extract Data on Normal PDF with bounding box with only a part of one word selected.
     ####################################################################################################
-    target_text = "Lo"
-
     request = ExtractDataRequest(
         filename=TEST_CLIPPING_BEHAVIOR_PDF_KEY.name,
         page_number=1,
@@ -231,17 +229,14 @@ def test_clipping_behavior(test_client: TestClient, upload_test_pdf, upload_test
         format=FormatTypes.TEXT,
     )
     response = test_client.post("/api/V1/extract_data", content=request.model_dump_json())
-    assert response.status_code == 200
-    json_response = response.json()
-    assert "bbox" in json_response
-    assert json_response["text"] == target_text
+    assert response.status_code == 404
 
     target_text = "Lorem"
 
     request = ExtractDataRequest(
         filename=TEST_CLIPPING_BEHAVIOR_PDF_KEY.name,
         page_number=1,
-        bbox=BoundingBox(x0=315, y0=300, x1=465, y1=330),  # pixels
+        bbox=BoundingBox(x0=315, y0=300, x1=440, y1=330),  # pixels
         format=FormatTypes.TEXT,
     )
     response = test_client.post("/api/V1/extract_data", content=request.model_dump_json())