Repository: Xilinx/inference-server (language: python)

Issue:
Support FP16
# Summary of Changes

*  Add FP16 support

Closes #75 

# Motivation

Adds FP16 support to the server.

# Implementation

FP16 support is implemented with [half](http://half.sourceforge.net/), an implementation of the IEEE half-precision floating point data type. Enabling this primarily means supporting the type at the external interfaces where the server communicates to/from clients. For both gRPC and HTTP, the protocol saves the data as 32-bit floating point and the server will cast it to 16 bit internally and store at as such until responding to the client at which point it will be converted back to 32-bit float.

A separate issue will be raised to address the use of raw data contents for passing the FP16 data as is without a conversion to 32 bit float.

Edit: issue #78 tracks allowing the use of raw data contents


Discussion / hints:
test this please
Can one of the admins verify this patch?
ok to test
ok to test
add to whitelist

Build failed!

Build successful!


Fix patch:
diff --git a/src/amdinfer/clients/grpc_internal.cpp b/src/amdinfer/clients/grpc_internal.cpp
index a17e19bc..f8dfea63 100644
--- a/src/amdinfer/clients/grpc_internal.cpp
+++ b/src/amdinfer/clients/grpc_internal.cpp
@@ -185,8 +185,13 @@ struct SetOutputData {
       std::memcpy(data.data(), (*contents)->data(), size * sizeof(T));
       output->setData(std::move(data));
     } else {
-      if constexpr (util::is_any_v<T, int8_t, uint8_t, int16_t, uint16_t,
-                                   fp16>) {
+      if constexpr (std::is_same_v<T, fp16>) {
+        for (auto i = 0U; i < size; ++i) {
+          auto datum = static_cast<fp16>(contents[i]);
+          std::memcpy(&(data[i * sizeof(T)]), &(datum), sizeof(T));
+        }
+      } else if constexpr (util::is_any_v<T, int8_t, uint8_t, int16_t,
+                                          uint16_t>) {
         for (auto i = 0U; i < size; ++i) {
           std::memcpy(&(data[i * sizeof(T)]), &(contents[i]), sizeof(T));
         }


Test patch:
diff --git a/tests/download/models/resnet50.py b/tests/download/models/resnet50.py
index b8ce5acd..e0cf8f25 100644
--- a/tests/download/models/resnet50.py
+++ b/tests/download/models/resnet50.py
@@ -51,5 +51,10 @@ def get(args: argparse.Namespace):
             directory,
             "",
         )
+        models["onnx_resnet50_fp16"] = FloatOpenDownload(
+            "pt_resnet50v1.5_imagenet_224_224_0.4_4.9G_1.1_M2.4.zip",
+            directory,
+            "resnet50/float/resnet50_fp16.onnx",
+        )
 
     return models
diff --git a/tests/src/amdinfer/testing/gtest_fixtures.hpp b/tests/src/amdinfer/testing/gtest_fixtures.hpp
index 4d0baa5d..1c8b26d4 100644
--- a/tests/src/amdinfer/testing/gtest_fixtures.hpp
+++ b/tests/src/amdinfer/testing/gtest_fixtures.hpp
@@ -51,9 +51,7 @@ class GrpcFixture : public BaseFixture {
       BaseFixture::SetUp();
       server_.startGrpc(kDefaultGrpcPort);
       started_ = true;
-      while (!client_->serverLive()) {
-        std::this_thread::yield();
-      }
+      amdinfer::waitUntilServerReady(client_.get());
     }
   }
 
@@ -76,9 +74,7 @@ class HttpFixture : public BaseFixture {
     if (!client_->serverLive()) {
       server_.startHttp(kDefaultHttpPort);
       started_ = true;
-      while (!client_->serverLive()) {
-        std::this_thread::yield();
-      }
+      amdinfer::waitUntilServerReady(client_.get());
     }
   }
 
diff --git a/tests/workers/CMakeLists.txt b/tests/workers/CMakeLists.txt
index 3a938e21..f40db386 100644
--- a/tests/workers/CMakeLists.txt
+++ b/tests/workers/CMakeLists.txt
@@ -43,6 +43,10 @@ foreach(test ${tests})
   endif()
 endforeach()
 
+if(${AMDINFER_ENABLE_MIGRAPHX})
+  add_subdirectory(migraphx)
+endif()
+
 if(${AMDINFER_ENABLE_VITIS})
   add_subdirectory(xmodel)
 endif()
diff --git a/tests/workers/migraphx/CMakeLists.txt b/tests/workers/migraphx/CMakeLists.txt
new file mode 100644
index 00000000..5ddc25c1
--- /dev/null
+++ b/tests/workers/migraphx/CMakeLists.txt
@@ -0,0 +1,24 @@
+# Copyright 2022 Advanced Micro Devices, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(tests resnet50)
+
+amdinfer_add_system_tests("${tests}")
+
+foreach(test ${tests})
+  amdinfer_get_test_target(target ${test})
+  target_link_libraries(
+    ${target} PRIVATE opencv_core opencv_imgproc opencv_imgcodecs
+  )
+endforeach()
diff --git a/tests/workers/migraphx/test_resnet50.cpp b/tests/workers/migraphx/test_resnet50.cpp
new file mode 100644
index 00000000..5a8edbb4
--- /dev/null
+++ b/tests/workers/migraphx/test_resnet50.cpp
@@ -0,0 +1,216 @@
+// Copyright 2023 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>         // for max
+#include <array>             // for array
+#include <cstdint>           // for int8_t, uint64_t
+#include <initializer_list>  // for initializer_list
+#include <memory>            // for unique_ptr
+#include <opencv2/core.hpp>  // for int8_t
+#include <string>            // for string, allocator
+#include <vector>            // for vector
+
+#include "amdinfer/amdinfer.hpp"                       // for InferenceRequest
+#include "amdinfer/pre_post/image_preprocess.hpp"      // for ImagePreproces...
+#include "amdinfer/pre_post/resnet50_postprocess.hpp"  // for resnet50Postpr...
+#include "amdinfer/testing/get_path_to_asset.hpp"      // for getPathToAsset
+#include "amdinfer/testing/gtest_fixtures.hpp"         // for Message, Suite...
+
+namespace amdinfer {
+
+template <typename T>
+using Images = std::vector<std::vector<T>>;
+
+Images<float> preprocess_fp32(const std::vector<std::string>& paths) {
+  const std::array<float, 3> mean{0.485F, 0.456F, 0.406F};
+  const std::array<float, 3> std{4.367F, 4.464F, 4.444F};
+  const auto image_size = 224;
+  const auto convert_scale = 1 / 255.0;
+
+  amdinfer::pre_post::ImagePreprocessOptions<float, 3> options;
+  options.order = amdinfer::pre_post::ImageOrder::NCHW;
+  options.height = image_size;
+  options.width = image_size;
+  options.mean = mean;
+  options.std = std;
+  options.normalize = true;
+  options.convert_color = true;
+  options.color_code = cv::COLOR_BGR2RGB;
+  options.convert_type = true;
+  options.type = CV_32FC3;
+  options.convert_scale = convert_scale;
+  return amdinfer::pre_post::imagePreprocess(paths, options);
+}
+
+Images<fp16> preprocess_fp16(const std::vector<std::string>& paths) {
+  const std::array<fp16, 3> mean{static_cast<fp16>(0.485F),
+                                 static_cast<fp16>(0.456F),
+                                 static_cast<fp16>(0.406F)};
+  const std::array<fp16, 3> std{static_cast<fp16>(4.367F),
+                                static_cast<fp16>(4.464F),
+                                static_cast<fp16>(4.444F)};
+  const auto image_size = 224;
+  const auto convert_scale = 1 / 255.0;
+
+  amdinfer::pre_post::ImagePreprocessOptions<fp16, 3> options;
+  options.order = amdinfer::pre_post::ImageOrder::NCHW;
+  options.height = image_size;
+  options.width = image_size;
+  options.mean = mean;
+  options.std = std;
+  options.normalize = true;
+  options.convert_color = true;
+  options.color_code = cv::COLOR_BGR2RGB;
+  options.convert_type = true;
+  options.type = CV_16F;
+  options.convert_scale = convert_scale;
+  return amdinfer::pre_post::imagePreprocess(paths, options);
+}
+
+template <typename T>
+std::vector<int> postprocess(const amdinfer::InferenceResponseOutput& output,
+                             int k) {
+  return amdinfer::pre_post::resnet50Postprocess(
+    static_cast<T*>(output.getData()), output.getSize(), k);
+}
+
+std::string workerLoad(Client* client, const ParameterMap& parameters) {
+  return client->workerLoad("Migraphx", parameters);
+}
+
+template <typename T>
+std::vector<InferenceRequest> constructRequests(const Images<T>& images) {
+  std::vector<InferenceRequest> requests;
+  requests.reserve(images.size());
+
+  const std::initializer_list<uint64_t> shape = {224, 224, 3};
+
+  DataType type;
+  if constexpr (std::is_same_v<T, fp16>) {
+    type = DataType::Fp16;
+  } else {
+    type = DataType::Fp32;
+  }
+
+  for (const auto& image : images) {
+    requests.emplace_back();
+    // NOLINTNEXTLINE(google-readability-casting)
+    requests.back().addInputTensor((void*)image.data(), shape, type);
+  }
+
+  return requests;
+}
+
+template <typename T, int K>
+void validate(const std::vector<InferenceResponse>& responses,
+              const std::array<int, K>& golden) {
+  const auto k = golden.size();
+
+  for (const auto& response : responses) {
+    auto outputs = response.getOutputs();
+    ASSERT_EQ(outputs.size(), 1);
+    auto top_k = postprocess<T>(outputs[0], k);
+    for (auto j = 0U; j < k; ++j) {
+      EXPECT_EQ(top_k.at(j), golden.at(j));
+    }
+  }
+}
+
+void validateFp16(const std::vector<InferenceResponse>& responses) {
+  const std::array golden{259, 261, 260, 154, 157};
+  validate<fp16, 5>(responses, golden);
+}
+
+void validateFp32(const std::vector<InferenceResponse>& responses) {
+  const std::array golden{259, 261, 157, 260, 154};
+  validate<float, 5>(responses, golden);
+}
+
+void test_fp16(Client* client) {
+  if (!serverHasExtension(client, "migraphx")) {
+    GTEST_SKIP() << "MIGraphX support required from the server but not found";
+  }
+
+  const auto test_asset = getPathToAsset("asset_dog-3619020_640.jpg");
+  const auto model = getPathToAsset("onnx_resnet50_fp16");
+
+  amdinfer::ParameterMap parameters;
+  parameters.put("model", model);
+
+  auto images = preprocess_fp16({test_asset});
+  auto endpoint = workerLoad(client, parameters);
+  auto requests = constructRequests(images);
+  auto responses = inferAsyncOrdered(client, endpoint, requests);
+  validateFp16(responses);
+
+  client->workerUnload(endpoint);
+  waitUntilModelNotReady(client, endpoint);
+}
+
+void test_fp32(Client* client) {
+  if (!serverHasExtension(client, "migraphx")) {
+    GTEST_SKIP() << "MIGraphX support required from the server but not found";
+  }
+
+  const auto test_asset = getPathToAsset("asset_dog-3619020_640.jpg");
+  const auto model = getPathToAsset("onnx_resnet50");
+
+  amdinfer::ParameterMap parameters;
+  parameters.put("model", model);
+
+  auto images = preprocess_fp32({test_asset});
+  auto endpoint = workerLoad(client, parameters);
+  auto requests = constructRequests(images);
+  auto responses = inferAsyncOrdered(client, endpoint, requests);
+  validateFp32(responses);
+
+  client->workerUnload(endpoint);
+  waitUntilModelNotReady(client, endpoint);
+}
+
+#ifdef AMDINFER_ENABLE_GRPC
+// @pytest.mark.extensions(["migraphx"])
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST_F(GrpcFixture, WorkersMigraphxResnet50Fp16) { test_fp16(client_.get()); }
+
+// @pytest.mark.extensions(["migraphx"])
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST_F(GrpcFixture, WorkersMigraphxResnet50Fp32) { test_fp32(client_.get()); }
+#endif
+
+// @pytest.mark.extensions(["migraphx"])
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST_F(BaseFixture, WorkersMigraphxResnet50Fp16) {
+  NativeClient client(&server_);
+  test_fp16(&client);
+}
+
+// @pytest.mark.extensions(["migraphx"])
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST_F(BaseFixture, WorkersMigraphxResnet50Fp32) {
+  NativeClient client(&server_);
+  test_fp32(&client);
+}
+
+#ifdef AMDINFER_ENABLE_HTTP
+// @pytest.mark.extensions(["migraphx"])
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST_F(HttpFixture, WorkersMigraphxResnet50Fp16) { test_fp16(client_.get()); }
+
+// @pytest.mark.extensions(["migraphx"])
+// NOLINTNEXTLINE(cert-err58-cpp, cppcoreguidelines-owning-memory)
+TEST_F(HttpFixture, WorkersMigraphxResnet50Fp32) { test_fp32(client_.get()); }
+#endif
+
+}  // namespace amdinfer