Repository: PolusAI/nyxus (language: python)

Issue:
Cumulative PR: whole slide performance and Py library fixes



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index d47669ce..b5641cbc 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -12,7 +12,7 @@ jobs:
       MACOSX_DEPLOYMENT_TARGET: "10.15"
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-10.15]
+        os: [ubuntu-latest, macos-10.15, windows-latest]
         cibw_archs: ["auto"]
 
     steps:
@@ -30,7 +30,7 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==2.1.1
+          python -m pip install cibuildwheel==2.1.1 delvewheel
 
       - name: Build wheels for CPython 3.9
         run: |
@@ -42,6 +42,10 @@ jobs:
           CIBW_BEFORE_BUILD_MACOS: brew install libtiff llvm libomp
           CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix llvm)/bin:$PATH" CFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" CXXFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" LDFLAGS="-L /usr/local/lib -L/usr/local/opt/llvm/lib" CXX=${COMPILER}
           CIBW_BEFORE_BUILD_LINUX: yum -y install libtiff-devel
+          CIBW_ARCHS_WINDOWS: "auto64"
+          CIBW_ENVIRONMENT_WINDOWS: TIFF_INCLUDE_DIR="C:/Program Files (x86)/tiff/include" TIFF_LIBRARY_RELEASE="C:/Program Files (x86)/tiff/lib/tiff.lib" PATH="C:\\Program Files (x86)\\tiff\\bin;$PATH"
+          CIBW_BEFORE_BUILD_WINDOWS: install_win_prereq.bat
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
 
       - name: Build wheels for CPython 3.8
@@ -54,6 +58,10 @@ jobs:
           CIBW_BEFORE_BUILD_MACOS: brew install libtiff llvm libomp
           CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix llvm)/bin:$PATH" CFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" CXXFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" LDFLAGS="-L /usr/local/lib -L/usr/local/opt/llvm/lib" CXX=${COMPILER}
           CIBW_BEFORE_BUILD_LINUX: yum -y install libtiff-devel
+          CIBW_ARCHS_WINDOWS: "auto64"
+          CIBW_ENVIRONMENT_WINDOWS: TIFF_INCLUDE_DIR="C:/Program Files (x86)/tiff/include" TIFF_LIBRARY_RELEASE="C:/Program Files (x86)/tiff/lib/tiff.lib" PATH="C:\\Program Files (x86)\\tiff\\bin;$PATH"
+          CIBW_BEFORE_BUILD_WINDOWS: install_win_prereq.bat
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
 
       - name: Build wheels for CPython 3.7
@@ -66,6 +74,10 @@ jobs:
           CIBW_BEFORE_BUILD_MACOS: brew install libtiff llvm libomp
           CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix llvm)/bin:$PATH" CFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" CXXFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" LDFLAGS="-L /usr/local/lib -L/usr/local/opt/llvm/lib" CXX=${COMPILER}
           CIBW_BEFORE_BUILD_LINUX: yum -y install libtiff-devel
+          CIBW_ARCHS_WINDOWS: "auto64"
+          CIBW_ENVIRONMENT_WINDOWS: TIFF_INCLUDE_DIR="C:/Program Files (x86)/tiff/include" TIFF_LIBRARY_RELEASE="C:/Program Files (x86)/tiff/lib/tiff.lib" PATH="C:\\Program Files (x86)\\tiff\\bin;$PATH"
+          CIBW_BEFORE_BUILD_WINDOWS: install_win_prereq.bat
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
 
       - name: Build wheels for CPython 3.6
@@ -78,4 +90,8 @@ jobs:
           CIBW_BEFORE_BUILD_MACOS: brew install libtiff llvm libomp
           CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix llvm)/bin:$PATH" CFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" CXXFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" LDFLAGS="-L /usr/local/lib -L/usr/local/opt/llvm/lib" CXX=${COMPILER}
           CIBW_BEFORE_BUILD_LINUX: yum -y install libtiff-devel
+          CIBW_ARCHS_WINDOWS: "auto64"
+          CIBW_ENVIRONMENT_WINDOWS: TIFF_INCLUDE_DIR="C:/Program Files (x86)/tiff/include" TIFF_LIBRARY_RELEASE="C:/Program Files (x86)/tiff/lib/tiff.lib" PATH="C:\\Program Files (x86)\\tiff\\bin;$PATH"
+          CIBW_BEFORE_BUILD_WINDOWS: install_win_prereq.bat
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
\ No newline at end of file
diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml
index b6eef1e2..05b26732 100644
--- a/.github/workflows/publish_pypi.yml
+++ b/.github/workflows/publish_pypi.yml
@@ -14,7 +14,7 @@ jobs:
       MACOSX_DEPLOYMENT_TARGET: "10.15"
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-10.15]
+        os: [ubuntu-latest, macos-10.15, windows-latest]
         cibw_archs: ["auto"]
 
     steps:
@@ -28,11 +28,9 @@ jobs:
         with:
           python-version: '3.8'
 
-
-
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==2.1.1
+          python -m pip install cibuildwheel==2.1.1 delvewheel
 
       - name: Build wheels for CPython 3.9
         run: |
@@ -44,6 +42,10 @@ jobs:
           CIBW_BEFORE_BUILD_MACOS: brew install libtiff llvm libomp
           CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix llvm)/bin:$PATH" CFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" CXXFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" LDFLAGS="-L /usr/local/lib -L/usr/local/opt/llvm/lib" CXX=${COMPILER}
           CIBW_BEFORE_BUILD_LINUX: yum -y install libtiff-devel
+          CIBW_ARCHS_WINDOWS: "auto64"
+          CIBW_ENVIRONMENT_WINDOWS: TIFF_INCLUDE_DIR="C:/Program Files (x86)/tiff/include" TIFF_LIBRARY_RELEASE="C:/Program Files (x86)/tiff/lib/tiff.lib" PATH="C:\\Program Files (x86)\\tiff\\bin;$PATH"
+          CIBW_BEFORE_BUILD_WINDOWS: install_win_prereq.bat
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
 
       - name: Build wheels for CPython 3.8
@@ -56,6 +58,10 @@ jobs:
           CIBW_BEFORE_BUILD_MACOS: brew install libtiff llvm libomp
           CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix llvm)/bin:$PATH" CFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" CXXFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" LDFLAGS="-L /usr/local/lib -L/usr/local/opt/llvm/lib" CXX=${COMPILER}
           CIBW_BEFORE_BUILD_LINUX: yum -y install libtiff-devel
+          CIBW_ARCHS_WINDOWS: "auto64"
+          CIBW_ENVIRONMENT_WINDOWS: TIFF_INCLUDE_DIR="C:/Program Files (x86)/tiff/include" TIFF_LIBRARY_RELEASE="C:/Program Files (x86)/tiff/lib/tiff.lib" PATH="C:\\Program Files (x86)\\tiff\\bin;$PATH"
+          CIBW_BEFORE_BUILD_WINDOWS: install_win_prereq.bat
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
 
       - name: Build wheels for CPython 3.7
@@ -68,6 +74,10 @@ jobs:
           CIBW_BEFORE_BUILD_MACOS: brew install libtiff llvm libomp
           CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix llvm)/bin:$PATH" CFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" CXXFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" LDFLAGS="-L /usr/local/lib -L/usr/local/opt/llvm/lib" CXX=${COMPILER}
           CIBW_BEFORE_BUILD_LINUX: yum -y install libtiff-devel
+          CIBW_ARCHS_WINDOWS: "auto64"
+          CIBW_ENVIRONMENT_WINDOWS: TIFF_INCLUDE_DIR="C:/Program Files (x86)/tiff/include" TIFF_LIBRARY_RELEASE="C:/Program Files (x86)/tiff/lib/tiff.lib" PATH="C:\\Program Files (x86)\\tiff\\bin;$PATH"
+          CIBW_BEFORE_BUILD_WINDOWS: install_win_prereq.bat
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
 
       - name: Build wheels for CPython 3.6
@@ -80,6 +90,10 @@ jobs:
           CIBW_BEFORE_BUILD_MACOS: brew install libtiff llvm libomp
           CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix llvm)/bin:$PATH" CFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" CXXFLAGS="-I /usr/local/include -I/usr/local/opt/llvm/include" LDFLAGS="-L /usr/local/lib -L/usr/local/opt/llvm/lib" CXX=${COMPILER}
           CIBW_BEFORE_BUILD_LINUX: yum -y install libtiff-devel
+          CIBW_ARCHS_WINDOWS: "auto64"
+          CIBW_ENVIRONMENT_WINDOWS: TIFF_INCLUDE_DIR="C:/Program Files (x86)/tiff/include" TIFF_LIBRARY_RELEASE="C:/Program Files (x86)/tiff/lib/tiff.lib" PATH="C:\\Program Files (x86)\\tiff\\bin;$PATH"
+          CIBW_BEFORE_BUILD_WINDOWS: install_win_prereq.bat
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
 
       - name: Install Dependencies
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1165de9d..1cd6899f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,8 +7,20 @@ set(CMAKE_CXX_STANDARD_REQUIRED True)
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /permissive- ")
+	#==== Explicit check for LibTiff, needed for Windows
+	if (NOT DEFINED ${TIFF_INCLUDE_DIR})
+		set (TIFF_INCLUDE_DIR $ENV{TIFF_INCLUDE_DIR})
+		message(STATUS "TIFF_INCLUDE_DIR " ${TIFF_INCLUDE_DIR})
+	endif()
+
+	if (NOT DEFINED ${TIFF_LIBRARY_RELEASE})
+		set (TIFF_LIBRARY_RELEASE $ENV{TIFF_LIBRARY_RELEASE})
+		message(STATUS "TIFF_LIBRARY_RELEASE " ${TIFF_LIBRARY_RELEASE})
+	endif()
 endif()
 
+
+
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release)
 endif()
diff --git a/install_win_nyxus.bat b/install_win_nyxus.bat
new file mode 100644
index 00000000..e331b798
--- /dev/null
+++ b/install_win_nyxus.bat
@@ -0,0 +1,12 @@
+echo off
+set PATH=C:/Program Files (x86)/tiff/bin;%PATH%
+set TIFF_INCLUDE_DIR=C:/Program Files (x86)/tiff/include
+set TIFF_LIBRARY_RELEASE=C:/Program Files (x86)/tiff/lib/tiff.lib
+python -m pip install delvewheel
+python setup.py bdist_wheel -d dist
+pushd dist
+for %%j in (*.whl) do (
+  delvewheel repair %%j
+  pip install wheelhouse\%%j 
+  ) 
+popd
\ No newline at end of file
diff --git a/install_win_prereq.bat b/install_win_prereq.bat
new file mode 100644
index 00000000..23d4613f
--- /dev/null
+++ b/install_win_prereq.bat
@@ -0,0 +1,13 @@
+@echo off
+Rem Download and Install libtiff
+@echo on
+curl https://download.osgeo.org/libtiff/tiff-4.3.0.zip -o libtiff.zip
+tar -xf libtiff.zip
+pushd tiff-4.3.0
+mkdir build_local
+pushd build_local
+cmake ..
+cmake --build . --config Release
+cmake --install . 
+popd
+popd
\ No newline at end of file
diff --git a/src/nyx/python/new_bindings_py.cpp b/src/nyx/python/new_bindings_py.cpp
index a84c198b..03f2cdd6 100644
--- a/src/nyx/python/new_bindings_py.cpp
+++ b/src/nyx/python/new_bindings_py.cpp
@@ -27,8 +27,8 @@ void initialize_environment(
     const std::vector<std::string> &features,
     float neighbor_distance,
     float pixels_per_micron,
-    uint n_reduce_threads,
-    uint n_loader_threads)
+    uint32_t n_reduce_threads,
+    uint32_t n_loader_threads)
 {
     theEnvironment.desiredFeatures = features;
     theEnvironment.set_pixel_distance(static_cast<int>(neighbor_distance));


Test patch:
diff --git a/.github/workflows/build_and_test_mac.yml b/.github/workflows/build_and_test_mac.yml
index 6f528fea..3fdbbcbb 100644
--- a/.github/workflows/build_and_test_mac.yml
+++ b/.github/workflows/build_and_test_mac.yml
@@ -30,7 +30,7 @@ jobs:
 
     - name: Update Path
       run: |
-        echo "brew --prefix llvm)/bin" >> $GITHUB_PATH
+        echo "$(brew --prefix llvm)/bin" >> $GITHUB_PATH
 
     - name: Configure CMake
       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBUILD_CLI=ON
@@ -44,8 +44,8 @@ jobs:
       run: ./tests/runAllTests  -C ${{env.BUILD_TYPE}}
       
 
-  Build_And_Run_UnitTest:
-    name: Build and Run Unit Tests
+  Build_and_Run_PyTest:
+    name: Build and Run PyTest
     runs-on: macos-10.15
 
     steps:
@@ -58,7 +58,7 @@ jobs:
 
     - name: Update Path
       run: |
-        echo "brew --prefix llvm)/bin" >> $GITHUB_PATH
+        echo "$(brew --prefix llvm)/bin" >> $GITHUB_PATH
 
     - name: Setup Python
       uses: actions/setup-python@v2
@@ -66,12 +66,12 @@ jobs:
         python-version: "3.x"
 
     - name: Install Dependencies
-      run: python3 -m pip install --upgrade wheel numpy pandas
+      run: python3 -m pip install --upgrade wheel numpy pandas pytest
 
     - name: Install Nyxus
       working-directory: ${{github.workspace}}
       run: python3 setup.py install
 
-    - name: Run Unit Tests
+    - name: Run PyTest
       working-directory: ${{github.workspace}}
-      run: python3 -m unittest discover tests/python/
\ No newline at end of file
+      run: python3 -m pytest tests/python/
\ No newline at end of file
diff --git a/.github/workflows/build_and_test_ubuntu.yml b/.github/workflows/build_and_test_ubuntu.yml
index d1aa3a45..5240275e 100644
--- a/.github/workflows/build_and_test_ubuntu.yml
+++ b/.github/workflows/build_and_test_ubuntu.yml
@@ -34,8 +34,8 @@ jobs:
       run: ./tests/runAllTests  -C ${{env.BUILD_TYPE}}
       
 
-  Build_and_Run_UnitTest:
-    name: Build and Run Unit Tests
+  Build_and_Run_PyTest:
+    name: Build and Run PyTest
     runs-on: ubuntu-latest
 
     steps:
@@ -52,12 +52,12 @@ jobs:
         python-version: "3.x"
 
     - name: Install Dependencies
-      run: python -m pip install --upgrade wheel numpy pandas
+      run: python -m pip install --upgrade wheel numpy pandas pytest
 
     - name: Install Nyxus
       working-directory: ${{github.workspace}}
       run: python3 setup.py install
 
-    - name: Run Unit Tests
+    - name: Run PyTest
       working-directory: ${{github.workspace}}
-      run: python3 -m unittest discover tests/python/
\ No newline at end of file
+      run: python3 -m pytest tests/python/
\ No newline at end of file
diff --git a/.github/workflows/build_and_test_windows.yml b/.github/workflows/build_and_test_windows.yml
new file mode 100644
index 00000000..e8f580aa
--- /dev/null
+++ b/.github/workflows/build_and_test_windows.yml
@@ -0,0 +1,79 @@
+name: Build And Run Test on Windows
+
+on:
+  push:
+    branches: [ main ]
+  pull_request: 
+  workflow_dispatch:
+
+env:
+  BUILD_TYPE: Release
+
+jobs:
+  Build_and_Run_GTest:
+    name: Build and Run GoogleTest
+    runs-on: windows-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        
+    - name: Install LibTiff
+      working-directory: ${{github.workspace}}
+      shell: cmd
+      run: install_win_prereq.bat
+
+    - name: Set variable
+      run: |
+        echo "TIFF_INCLUDE_DIR=C:/Program Files (x86)/tiff/include" >> $env:GITHUB_ENV
+        echo "TIFF_LIBRARY_RELEASE=C:/Program Files (x86)/tiff/lib/tiff.lib" >> $env:GITHUB_ENV
+ 
+    - name: Configure CMake
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DBUILD_CLI=ON
+
+    - name: Build
+      run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --parallel 2
+
+    - name: Run GTest Suite
+      working-directory: ${{github.workspace}}/build
+      # Execute tests.  
+      run: ./tests/${{env.BUILD_TYPE}}/runAllTests.exe
+      
+
+  Build_and_Run_PyTest:
+    name: Build and Run PyTest
+    runs-on: windows-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+              
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.x"
+
+    - name: Install Dependencies
+      run: python -m pip install --upgrade wheel numpy pandas pytest delvewheel
+
+    - name: Install LibTiff
+      working-directory: ${{github.workspace}}
+      shell: cmd
+      run: install_win_prereq.bat
+
+    - name: Set variable
+      run: |
+        echo "TIFF_INCLUDE_DIR=C:/Program Files (x86)/tiff/include" >> $env:GITHUB_ENV
+        echo "TIFF_LIBRARY_RELEASE=C:/Program Files (x86)/tiff/lib/tiff.lib" >> $env:GITHUB_ENV
+        echo "C:/Program Files (x86)/tiff/bin" >> $env:GITHUB_PATH
+ 
+    - name: Install Nyxus
+      working-directory: ${{github.workspace}}
+      shell: cmd
+      run: install_win_nyxus.bat
+
+    - name: Run PyTest
+      working-directory: ${{github.workspace}}
+      run: python3 -m pytest tests/python/
\ No newline at end of file
diff --git a/tests/python/__init__.py b/tests/python/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/tests/python/test_nyxus.py b/tests/python/test_nyxus.py
index a5966074..5bac564b 100644
--- a/tests/python/test_nyxus.py
+++ b/tests/python/test_nyxus.py
@@ -1,7 +1,4 @@
 import nyxus
-import unittest
 
-class TestNyxus(unittest.TestCase):
-
-    def test_import_module(self):
-        self.assertEqual(nyxus.__name__ , "nyxus")
\ No newline at end of file
+def test_import():
+    assert nyxus.__name__ == "nyxus" 
\ No newline at end of file