Repository: deepdoctection/deepdoctection (language: python)

Issue:
PyYAML version 6 can't build, upgrade to 6.0.1
**Bug** 💥
PyYAML 6.0.0 can't compile on Cython >=3.0.0, so `pip install deepdoctection` doesn't work.  Bumping to 6.0.1 will fix it.

**Expected behavior**  🧮 
 `pip install deepdoctection` to work

**Additional context** 🧬
If necessary add some context related to the problem:


https://github.com/yaml/pyyaml/pull/702
https://github.com/yaml/pyyaml/issues/601


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml
index 3eca61f4..a22531e8 100644
--- a/.github/workflows/style.yaml
+++ b/.github/workflows/style.yaml
@@ -9,13 +9,14 @@ on:
 
 jobs:
   lint:
-    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master'"
+    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master' || github.event_name == 'pull_request'"
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest]
         python: [3.8]
-        dll: ["torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu"]
+        dll: ["torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu"]
+        pip: [24.0]
     steps:
       - uses: actions/checkout@v2
 
@@ -26,19 +27,22 @@ jobs:
           architecture: x64
 
       - name: Install DL library
-        run: pip install ${{ matrix.dll }}
+        run: |
+             python -m pip install pip==${{ matrix.pip }}
+             pip install ${{ matrix.dll }}
 
       - name: Install dependencies
         run: |
-          python -m pip install pip==22.1.2
-          pip install -e ".[source-pt]"
+          python -m pip install wheel
+          pip install --no-build-isolation detectron2@git+https://github.com/facebookresearch/detectron2.git
+          pip install -e ".[pt]"
           pip install -e ".[dev, test]"
 
       - name: Run linter
         run: make lint
 
   black:
-    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master'"
+    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master' || github.event_name == 'pull_request'"
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -59,7 +63,7 @@ jobs:
           make black
 
   isort:
-    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master'"
+    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master' || github.event_name == 'pull_request'"
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -76,19 +80,18 @@ jobs:
 
       - name: Run isort
         run: |
-          pip install isort
-          isort --version
+          pip install isort==5.13.2
           make isort
-          if [ -n "$(git status --porcelain --untracked-files=no)" ]; then exit 1; else echo "All clear"; fi
 
   mypy:
-    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master'"
+    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master' || github.event_name == 'pull_request'"
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest]
         python: [3.8]
-        dll: ["torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu"]
+        dll: ["torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu"]
+        pip: [24.0]
     steps:
       - uses: actions/checkout@v2
 
@@ -100,14 +103,16 @@ jobs:
 
       - name: Install DL library
         run: |
-           python -m pip install pip==22.1.2
+           python -m pip install pip==${{ matrix.pip }}
            pip install ${{ matrix.dll }}
 
       - name: Install dependencies
         run: |
-          pip install -e ".[source-pt]"
+          python -m pip install wheel
+          pip install --no-build-isolation detectron2@git+https://github.com/facebookresearch/detectron2.git
+          pip install -e ".[pt]"
           pip install -e ".[dev, test]"
 
       - name: Run mypy
         run: |
-          mypy -p deepdoctection -p tests -p tests_d2
\ No newline at end of file
+          make analyze
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 9293b6f0..426fb942 100644
--- a/Makefile
+++ b/Makefile
@@ -72,13 +72,13 @@ install-jupyterlab-setup: check-venv
 	@echo "--> Done installing Jupyter Lab"
 
 install-kernel-dd: check-venv
-	@echo "--> Installing IPkernel setup and setup kernel deep-doctection"
+	@echo "--> Installing IPkernel setup and setup kernel deepdoctection"
 	pip install --user ipykernel
 	$(PYTHON) -m ipykernel install --user --name=deep-doc
 	@echo "--> Done installing kernel deep-doc"
 
 install-kernel-dd-mac: check-venv
-	@echo "--> Installing IPkernel setup and setup kernel deep-doctection"
+	@echo "--> Installing IPkernel setup and setup kernel deepdoctection"
 	pip install ipykernel
 	$(PYTHON) -m ipykernel install --name=deep-doc
 	@echo "--> Done installing kernel deep-doc"
diff --git a/README.md b/README.md
index d108146e..be9fd859 100644
--- a/README.md
+++ b/README.md
@@ -42,7 +42,9 @@ pipelines. Its core function does not depend on any specific deep learning libra
    Check this [notebook](https://github.com/deepdoctection/notebooks/blob/main/Analyzer_Configuration.ipynb) or the 
    [docs](https://deepdoctection.readthedocs.io/en/latest/tutorials/analyzer_configuration_notebook/) for more infos.
  - Document layout analysis and table recognition now runs with Torchscript (CPU) as well and Detectron2 is 
-   not required anymore for basic inference. 
+   not required anymore for basic inference.
+ - [**new**] More angle predictors for determining the rotation of a document based on Tesseract and DocTr 
+   (not contained in the built-in Analyzer).
 
 **deep**doctection provides on top of that methods for pre-processing inputs to models like cropping or resizing and to 
 post-process results, like validating duplicate outputs, relating words to detected layout segments or ordering words 
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 544adb6d..fbab1dcc 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -4,7 +4,7 @@
 #
 #    pip-compile --extra=docs --output-file=docs/requirements.txt setup.py
 #
-accelerate==0.25.0
+accelerate==0.29.1
     # via deepdoctection (setup.py)
 attrs==22.2.0
     # via jsonlines
@@ -14,7 +14,7 @@ botocore==1.29.35
     # via
     #   boto3
     #   s3transfer
-catalogue==2.0.7
+catalogue==2.0.10
     # via deepdoctection (setup.py)
 certifi==2022.12.7
     # via requests
@@ -59,7 +59,7 @@ importlib-metadata==5.2.0
     #   deepdoctection (setup.py)
     #   markdown
     #   mkdocs
-jdeskew==0.2.0
+jdeskew==0.2.2
     # via deepdoctection (setup.py)
 jinja2==3.0.3
     # via
@@ -72,11 +72,11 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-jsonlines==3.0.0
+jsonlines==3.1.0
     # via deepdoctection (setup.py)
 lxml==4.9.2
     # via deepdoctection (setup.py)
-lxml-stubs==0.4.0
+lxml-stubs==0.5.1
     # via deepdoctection (setup.py)
 markdown==3.3.7
     # via
@@ -126,7 +126,7 @@ numpy==1.24.0
     #   deepdoctection (setup.py)
     #   jdeskew
     #   msgpack-numpy
-    #   opencv-python
+    #   opencv-python-headless
     #   tensorpack
     #   transformers
 nvidia-cublas-cu12==12.1.3.1
@@ -160,7 +160,7 @@ nvidia-nvjitlink-cu12==12.3.101
     #   nvidia-cusparse-cu12
 nvidia-nvtx-cu12==12.1.105
     # via torch
-opencv-python==4.5.4.60
+opencv-python-headless==4.9.0.80
     # via jdeskew
 packaging==21.3
     # via
@@ -197,7 +197,7 @@ python-dateutil==2.8.2
     # via
     #   botocore
     #   ghp-import
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   accelerate
     #   deepdoctection (setup.py)
diff --git a/requirements.txt b/requirements.txt
index dda2595a..e56d7da1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@
 #
 attrs==21.4.0
     # via jsonlines
-catalogue==2.0.7
+catalogue==2.0.10
     # via deepdoctection (setup.py)
 certifi==2021.10.8
     # via requests
@@ -20,9 +20,9 @@ huggingface-hub==0.17.2
     # via deepdoctection (setup.py)
 idna==3.3
     # via requests
-importlib-metadata==4.11.2
+importlib-metadata==7.1.0
     # via deepdoctection (setup.py)
-jsonlines==3.0.0
+jsonlines==3.1.0
     # via deepdoctection (setup.py)
 mock==4.0.3
     # via deepdoctection (setup.py)
@@ -40,7 +40,7 @@ pyparsing==3.0.7
     # via packaging
 pypdf==3.17.3
     # via deepdoctection (setup.py)
-pyyaml==6.0
+pyyaml==6.0.1
     # via
     #   deepdoctection (setup.py)
     #   huggingface-hub
diff --git a/setup.py b/setup.py
index 0ebefd4f..58302a5b 100644
--- a/setup.py
+++ b/setup.py
@@ -42,10 +42,10 @@ def get_version():
 _DEPS = [
     # the minimum requirements to run pipelines without considering DL models specific dependencies
     "apted==1.0.3",
-    "catalogue==2.0.7",
+    "catalogue==2.0.10",
     "distance==0.1.3",
     "huggingface_hub>=0.12.0",
-    "importlib-metadata>=4.11.2",
+    "importlib-metadata>=5.0.0",
     "jsonlines==3.1.0",
     "lxml>=4.9.1",
     "mock==4.0.3",
@@ -213,7 +213,6 @@ def deps_list(*pkgs: str):
     "docs": docs_deps,
     "dev": dev_deps,
     "test": test_deps,
-    "hf": pt_deps,
 }
 
 setup(


Test patch:
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 74de73a5..9eab5c3d 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -14,41 +14,103 @@ permissions:
 
 jobs:
   tests:
-    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master'"
+    if: "contains(github.event.head_commit.message, '[force ci]') || github.ref == 'refs/heads/master' || github.event_name == 'pull_request'"
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: true
       matrix:
         include:
-          - { name: '3.8-basic-tf', python: '3.8', os: ubuntu-20.04,
-              dll: tensorflow==2.9, extras: ,
+          - { name: '3.8-basic-tf',
+              python: '3.8',
+              pip: '24.0',
+              os: ubuntu-20.04,
+              dll: tensorflow==2.9,
+              detectron2: false,
+              extras: ,
+              test: test,
               test_cases: test-basic }
-          - { name: '3.8-basic-pt', python: '3.8', os: ubuntu-20.04,
+          - { name: '3.8-basic-pt',
+              python: '3.8',
+              pip: '24.0',
+              os: ubuntu-20.04,
               dll: 'torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu',
+              detectron2: false,
               extras: ,
+              test: test,
               test_cases: test-basic-pt }
-          - { name: '3.8-tf', python: '3.8', os: ubuntu-20.04,
-              dll: tensorflow==2.9, extras: tf,
+          - { name: '3.8-tf',
+              python: '3.8',
+              pip: '24.0',
+              os: ubuntu-20.04,
+              dll: tensorflow==2.9,
+              detectron2: false,
+              extras: tf,
+              test: test,
               test_cases: test-tf }
-          - { name: '3.8-pt', python: '3.8', os: ubuntu-20.04,
+          - { name: '3.8-pt',
+              python: '3.8',
+              pip: '24.0',
+              os: ubuntu-20.04,
               dll: 'torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu',
-              extras: source-pt,
+              detectron2: true,
+              extras: pt,
+              test: test,
               test_cases: test-pt }
-          - { name: '3.10-basic-tf', python: '3.10', os: ubuntu-22.04,
-              dll: tensorflow==2.9, extras: ,
+          - { name: '3.10-basic-tf',
+              python: '3.10',
+              pip: '24.0',
+              os: ubuntu-22.04,
+              dll: tensorflow==2.9,
+              detectron2: false,
+              extras: ,
+              test: test,
               test_cases: test-basic }
-          - { name: '3.10-basic-pt', python: '3.10', os: ubuntu-22.04,
+          - { name: '3.10-basic-pt',
+              python: '3.10',
+              pip: '24.0',
+              os: ubuntu-22.04,
               dll: 'torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu',
+              detectron2: false,
               extras: ,
+              test: test,
               test_cases: test-basic-pt }
-          - { name: '3.10-tf', python: '3.10', os: ubuntu-22.04,
-              dll: tensorflow==2.9, extras: tf,
+          - { name: '3.10-tf',
+              python: '3.10',
+              pip: '24.0',
+              os: ubuntu-22.04,
+              dll: tensorflow==2.9,
+              detectron2: false,
+              extras: tf,
+              test: test,
               test_cases: test-tf }
-          - { name: '3.10-pt', python: '3.10', os: ubuntu-22.04,
+          - { name: '3.10-pt',
+              python: '3.10',
+              pip: '24.0',
+              os: ubuntu-22.04,
+              dll: 'torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu',
+              detectron2: true,
+              extras: pt,
+              test: test,
+              test_cases: test-pt }
+          - { name: '3.11-pt',
+              python: '3.11',
+              pip: '24.0',
+              os: ubuntu-22.04,
               dll: 'torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu',
-              extras: source-pt,
+              detectron2: true,
+              extras: pt,
+              test: test,
               test_cases: test-pt }
+          #- { name: '3.12-pt',
+          #    python: '3.12',
+          #    pip: '24.0',
+          #    os: ubuntu-22.04,
+          #    dll: 'torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu',
+          #    detectron2: true,
+          #    extras: pt,
+          #    test: test,
+          #    test_cases: test-pt }
 
     steps:
       - uses: actions/checkout@v3
@@ -59,7 +121,12 @@ jobs:
           python-version: ${{ matrix.python }}
 
       - name: Upgrade pip
-        run: python3 -m pip install pip==22.1.2
+        run: python3 -m pip install pip==${{ matrix.pip }} && 
+             python3 -m pip install wheel
+
+      #- name: Install setuptools
+      #  if: matrix.python == '3.12'
+      #  run: python3 -m pip install setuptools
 
       - name: Install libcap
         run: sudo apt-get install build-essential libcap-dev
@@ -79,6 +146,10 @@ jobs:
       - name: cd
         run: cd deepdoctection
 
+      - name: Install detectron2
+        if: matrix.detectron2
+        run: pip install --no-build-isolation detectron2@git+https://github.com/facebookresearch/detectron2.git
+
       - name: Test deepdoctection package installation
         if: matrix.extras == ''
         run: pip install "."
@@ -88,7 +159,7 @@ jobs:
         run:  pip install ".[${{ matrix.extras }}]"
 
       - name: Install test suite
-        run: pip install pytest pytest-cov
+        run: pip install ".[${{ matrix.test }}]"
 
       - name: Run tests
         run: make ${{ matrix.test_cases }}
\ No newline at end of file
diff --git a/tests/mapper/data.py b/tests/mapper/data.py
index 24ee883f..05843866 100644
--- a/tests/mapper/data.py
+++ b/tests/mapper/data.py
@@ -668,8 +668,8 @@ class DatapointCoco:
     """
 
     dp = _SAMPLE_COCO
-    white_image: ImageType = np.ones((794, 596, 3), dtype=np.int32) * 255  # type: ignore
-    white_image_string = convert_np_array_to_b64(white_image)
+    white_image = np.ones((794, 596, 3), dtype=np.uint8) * 255
+    white_image_string = convert_np_array_to_b64(white_image)  # type: ignore
     categories = {
         "1": LayoutType.text,
         "2": LayoutType.title,
@@ -686,7 +686,7 @@ def get_white_image(self, path: str, type_id: str = "np") -> Optional[Union[str,
         """
         if path == self.dp["file_name"]:
             if type_id == "np":
-                return self.white_image
+                return self.white_image # type: ignore
             return self.white_image_string
         return None
 
@@ -737,8 +737,8 @@ class DatapointPubtabnet:
     categories = {"1": LayoutType.cell, "2": TableType.item, "3": LayoutType.table, "4": LayoutType.word}
     categories_as_names = {v: k for k, v in categories.items()}
     first_ann_box = Box(475, 162, 10, 9)
-    white_image: ImageType = np.ones((1334, 996, 3), dtype=np.int32) * 255  # type: ignore
-    white_image_string = convert_np_array_to_b64(white_image)
+    white_image = np.ones((1334, 996, 3), dtype=np.uint8) * 255
+    white_image_string = convert_np_array_to_b64(white_image)  # type: ignore
 
     def get_white_image(self, path: str, type_id: str = "np") -> Union[str, ImageType]:
         """
@@ -747,7 +747,7 @@ def get_white_image(self, path: str, type_id: str = "np") -> Union[str, ImageTyp
         """
         assert path is not None
         if type_id == "np":
-            return self.white_image
+            return self.white_image  # type: ignore
         return self.white_image_string
 
     def get_width(self) -> float: