Repository: triton-inference-server/server
Pull request: http on windows
Related to triton-inference-server/server#3130

PR for third_party: https://github.com/triton-inference-server/third_party/pull/9

Problem statement:
Support for HTTP endpoint on windows
**Describe the solution you'd like**
Is it possible for Windows version of Triton to have HTTP server, like the Linux version does? Currently, only GRPC endpoint is supported. 

**Additional context**
Triton uses libevhtp in HTTP server implementation, and vcpkg seems to lack libevhtp for Windows. 
Is that the only blocker for having HTTP endpoint on Windows version of Triton ? Would it help if MinGW based vcpkg was created (it seems that libevhtp can't be build with MSVC, only MinGW on Windows) ?

Hints from issue discussion:
Triton likely needs a windows-native port of libevhtp. We suspect that it is just a matter of using the appropriate socket API for windows. Other than libevhtp there is no other blocker for getting HTTP support on windows. At this time we don't have a schedule for when we will be working on windows HTTP.

Commit message: dockerfile polishing

Patch to review:
diff --git a/Dockerfile.win10.min b/Dockerfile.win10.min
index 50285abd53..8f13b3f79e 100644
--- a/Dockerfile.win10.min
+++ b/Dockerfile.win10.min
@@ -58,11 +58,13 @@ ARG VS_INSTALL_PATH_WP="C:\BuildTools"
 RUN powershell.exe Start-Process -FilePath vs_buildtools.exe -ArgumentList "--wait","--quiet","--norestart","--nocache","--installPath","%VS_INSTALL_PATH_WP%","--channelUri","C:\tmp\doesnotexist.chman","--add","Microsoft.VisualStudio.Workload.VCTools`;includeRecommended","--add","Microsoft.Component.MSBuild" -Wait -PassThru
 
 WORKDIR /
-RUN git clone --depth=1 --single-branch -b 2021.04.30 https://github.com/microsoft/vcpkg.git
+# vcpkg 2021.05.12 does not contain b64 port, so checking out master branch here
+# should be switched to next vcpkg tag when one is created
+RUN git clone --depth=1 --single-branch -b master https://github.com/microsoft/vcpkg.git
 WORKDIR /vcpkg
 RUN bootstrap-vcpkg.bat
 RUN vcpkg.exe update
-RUN vcpkg.exe install openssl:x64-windows openssl-windows:x64-windows rapidjson:x64-windows re2:x64-windows boost-interprocess:x64-windows boost-stacktrace:x64-windows zlib:x64-windows
+RUN vcpkg.exe install openssl:x64-windows openssl-windows:x64-windows rapidjson:x64-windows re2:x64-windows boost-interprocess:x64-windows boost-stacktrace:x64-windows zlib:x64-windows pthread:x64-windows b64:x64-windows
 RUN vcpkg.exe integrate install
 
 ARG CUDA_MAJOR=11
diff --git a/src/servers/CMakeLists.txt b/src/servers/CMakeLists.txt
index ad07fa3adc..d05dd3a3c5 100644
--- a/src/servers/CMakeLists.txt
+++ b/src/servers/CMakeLists.txt
@@ -146,9 +146,16 @@ if(${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR ${TRITON_ENABLE_SAGEMAKE
     PRIVATE libevhtp::evhtp
     PRIVATE protobuf::libprotobuf
     PRIVATE ${RE2_LIBRARY}
-    PRIVATE b64
-    PRIVATE z
   )
+
+  if (WIN32)
+    find_library(B64_LIBRARY NAMES b64)
+    find_library(ZLIB_LIBRARY NAMES zlib)
+    list(APPEND HTTP_ENDPOINT_LIBRARIES ${B64_LIBRARY} ${ZLIB_LIBRARY})
+  else()
+    list(APPEND HTTP_ENDPOINT_LIBRARIES b64 z)
+  endif()
+
 endif() # TRITON_ENABLE_HTTP || TRITON_ENABLE_METRICS || TRITON_ENABLE_SAGEMAKER
 
 #
@@ -219,6 +226,7 @@ add_executable(
   ${TRACING_OBJECTS}
   $<TARGET_OBJECTS:proto-library>
 )
+
 set_property(TARGET main PROPERTY OUTPUT_NAME tritonserver)
 if(${TRITON_ENABLE_GPU})
   target_include_directories(main PRIVATE ${CUDA_INCLUDE_DIRS})
diff --git a/src/servers/http_server.cc b/src/servers/http_server.cc
index 2fb451166b..6b71f71195 100644
--- a/src/servers/http_server.cc
+++ b/src/servers/http_server.cc
@@ -24,6 +24,7 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#define NOMINMAX
 #include "src/servers/http_server.h"
 
 #include <event2/buffer.h>
@@ -67,11 +68,13 @@ HTTPServer::Start()
     evhtp_set_gencb(htp_, HTTPServer::Dispatch, this);
     evhtp_use_threads_wexit(htp_, NULL, NULL, thread_cnt_, NULL);
     evhtp_bind_socket(htp_, "0.0.0.0", port_, 1024);
+
     // Set listening event for breaking event loop
-    evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, fds_);
+    evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, (intptr_t*)fds_);
     break_ev_ = event_new(evbase_, fds_[0], EV_READ, StopCallback, evbase_);
     event_add(break_ev_, NULL);
     worker_ = std::thread(event_base_loop, evbase_, 0);
+
     return nullptr;
   }
 
@@ -84,7 +87,7 @@ HTTPServer::Stop()
 {
   if (worker_.joinable()) {
     // Notify event loop to break via fd write
-    send(fds_[1], &evbase_, sizeof(event_base*), 0);
+    send(fds_[1], (const char*) &evbase_, sizeof(event_base*), 0);
     worker_.join();
     event_free(break_ev_);
     evutil_closesocket(fds_[0]);
@@ -100,7 +103,7 @@ HTTPServer::Stop()
 }
 
 void
-HTTPServer::StopCallback(int sock, short events, void* arg)
+HTTPServer::StopCallback(evutil_socket_t sock, short events, void* arg)
 {
   struct event_base* base = (struct event_base*)arg;
   event_base_loopbreak(base);
@@ -834,7 +837,7 @@ EVBufferToJson(
   std::vector<char> json_buffer;
 
   // No need to memcpy when number of iovecs is 1
-  if ((n > 0) and (v[0].iov_len >= remaining_length)) {
+  if ((n > 0) && (v[0].iov_len >= remaining_length)) {
     json_base = static_cast<char*>(v[0].iov_base);
     if (v[0].iov_len > remaining_length) {
       v[0].iov_base = static_cast<void*>(json_base + remaining_length);
diff --git a/src/servers/http_server.h b/src/servers/http_server.h
index 5e5d2af4ea..00d670af9f 100644
--- a/src/servers/http_server.h
+++ b/src/servers/http_server.h
@@ -64,7 +64,7 @@ class HTTPServer {
  protected:
   virtual void Handle(evhtp_request_t* req) = 0;
 
-  static void StopCallback(int sock, short events, void* arg);
+  static void StopCallback(evutil_socket_t sock, short events, void* arg);
 
   int32_t port_;
   int thread_cnt_;
@@ -72,7 +72,7 @@ class HTTPServer {
   evhtp_t* htp_;
   struct event_base* evbase_;
   std::thread worker_;
-  int fds_[2];
+  evutil_socket_t fds_[2];
   event* break_ev_;
 };
 
diff --git a/src/servers/main.cc b/src/servers/main.cc
index 66aa1c5b51..ead8187b02 100644
--- a/src/servers/main.cc
+++ b/src/servers/main.cc
@@ -24,6 +24,11 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#ifdef _WIN32
+#define NOMINMAX
+#pragma comment(lib, "ws2_32.lib")
+#endif
+
 #ifndef _WIN32
 #include <getopt.h>
 #include <unistd.h>
@@ -702,6 +707,11 @@ StartEndpoints(
     const std::shared_ptr<nvidia::inferenceserver::SharedMemoryManager>&
         shm_manager)
 {
+#ifdef _WIN32
+  WSADATA wsaData;
+  WSAStartup(MAKEWORD(2,2), &wsaData);
+#endif
+
 #ifdef TRITON_ENABLE_GRPC
   // Enable GRPC endpoints if requested...
   if (allow_grpc_) {
@@ -806,6 +816,9 @@ StopEndpoints()
   }
 #endif  // TRITON_ENABLE_SAGEMAKER
 
+#ifdef _WIN32
+  WSACleanup();
+#endif
   return ret;
 }
 


Code review comments:
diff_hunk: @@ -702,6 +707,11 @@ StartEndpoints(
     const std::shared_ptr<nvidia::inferenceserver::SharedMemoryManager>&
         shm_manager)
 {
+#ifdef _WIN32
+  WSADATA wsaData;
+  WSAStartup(MAKEWORD(2,2), &wsaData);
path: src/servers/main.cc
text: @user1:
Should check return value in case of error.

diff_hunk: @@ -24,6 +24,7 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#define NOMINMAX
path: src/servers/http_server.cc
text: @user1:
Should this be surrounded by `ifdef _WIN32` like what is in `main.cc`?

diff_hunk: @@ -58,9 +58,9 @@ ARG VS_INSTALL_PATH_WP="C:\BuildTools"
 RUN powershell.exe Start-Process -FilePath vs_buildtools.exe -ArgumentList "--wait","--quiet","--norestart","--nocache","--installPath","%VS_INSTALL_PATH_WP%","--channelUri","C:\tmp\doesnotexist.chman","--add","Microsoft.VisualStudio.Workload.VCTools`;includeRecommended","--add","Microsoft.Component.MSBuild" -Wait -PassThru
 
 WORKDIR /
-# change back to https://github.com/microsoft/vcpkg.git
-# once b64 PR is merged: https://github.com/microsoft/vcpkg/pull/19447
-RUN git clone --depth=1 --single-branch -b libb64 https://github.com/9cvele3/vcpkg.git
+# vcpkg 2021.05.12 does not contain b64 port, so checking out master branch here
+# should be switched to next vcpkg tag when one is created
+RUN git clone --depth=1 --single-branch -b master https://github.com/microsoft/vcpkg.git
path: Dockerfile.win10.min
text: @user1:
I think we should check out a commit after cloning so that the build won't be affected by the ToT changes without anticipation.