Add higher level tritonserver API

.github/workflows/tritonserver.yml

@@ -19,6 +19,6 @@ env:
 jobs:
   linux-x86_64:
     runs-on: ubuntu-20.04
-    container: nvcr.io/nvidia/tritonserver:23.04-py3
+    container: nvcr.io/nvidia/tritonserver:23.05-py3
     steps:
       - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions

CHANGELOG.md

@@ -1,5 +1,6 @@
 
- * Upgrade presets for DNNL 3.1.1, CPython 3.11.4, NumPy 1.25.0, SciPy 1.11.0, LLVM 16.0.6, ONNX Runtime 1.15.1, and their dependencies
+ * Map new higher-level C++ API of Triton Inference Server ([pull #1361](https://github.com/bytedeco/javacpp-presets/pull/1361))
+ * Upgrade presets for DNNL 3.1.1, CPython 3.11.4, NumPy 1.25.0, SciPy 1.11.0, LLVM 16.0.6, Triton Inference Server 2.34.0, ONNX Runtime 1.15.1, and their dependencies
 
 ### June 6, 2023 version 1.5.9
  * Virtualize `nvinfer1::IGpuAllocator` from TensorRT to allow customization ([pull #1367](https://github.com/bytedeco/javacpp-presets/pull/1367))

README.md

@@ -225,7 +225,7 @@ Each child module in turn relies by default on the included [`cppbuild.sh` scrip
  * TensorFlow 1.15.x  https://github.com/tensorflow/tensorflow
  * TensorFlow Lite 2.12.x  https://github.com/tensorflow/tensorflow
  * TensorRT 8.6.x  https://developer.nvidia.com/tensorrt
- * Triton Inference Server 2.33.x  https://developer.nvidia.com/nvidia-triton-inference-server
+ * Triton Inference Server 2.34.x  https://developer.nvidia.com/nvidia-triton-inference-server
  * The Arcade Learning Environment 0.8.x  https://github.com/mgbellemare/Arcade-Learning-Environment
  * DepthAI 2.21.x  https://github.com/luxonis/depthai-core
  * ONNX 1.14.x  https://github.com/onnx/onnx

platform/pom.xml

@@ -311,7 +311,7 @@
     <dependency>
       <groupId>org.bytedeco</groupId>
       <artifactId>tritonserver-platform</artifactId>
-      <version>2.33-${project.version}</version>
+      <version>2.34-${project.version}</version>
     </dependency>
 <!--    <dependency>-->
 <!--      <groupId>org.bytedeco</groupId>-->

tritonserver/README.md

@@ -23,7 +23,7 @@ Introduction
 ------------
 This directory contains the JavaCPP Presets module for:
 
- * Triton Inference Server 2.33.0  https://github.com/triton-inference-server/server
+ * Triton Inference Server 2.34.0  https://github.com/triton-inference-server/server
 
 Please refer to the parent README.md file for more detailed information about the JavaCPP Presets.
 
@@ -51,17 +51,17 @@ This sample intends to show how to call the Java-mapped C API of Triton to execu
 
  1. Get the source code of Triton Inference Server to prepare the model repository:
 ```bash
- $ wget https://github.com/triton-inference-server/server/archive/refs/tags/v2.33.0.tar.gz
- $ tar zxvf v2.33.0.tar.gz
- $ cd server-2.33.0/docs/examples/model_repository
+ $ wget https://github.com/triton-inference-server/server/archive/refs/tags/v2.34.0.tar.gz
+ $ tar zxvf v2.34.0.tar.gz
+ $ cd server-2.34.0/docs/examples/model_repository
  $ mkdir models
  $ cd models; cp -a ../simple .
 ```
 Now, this `models` directory will be our model repository.
 
  2. Start the Docker container to run the sample (assuming we are under the `models` directory created above):
 ```bash
- $ docker run -it --gpus=all -v $(pwd):/workspace nvcr.io/nvidia/tritonserver:23.04-py3 bash
+ $ docker run -it --gpus=all -v $(pwd):/workspace nvcr.io/nvidia/tritonserver:23.05-py3 bash
  $ apt update
  $ apt install -y openjdk-11-jdk
  $ wget https://archive.apache.org/dist/maven/maven-3/3.8.4/binaries/apache-maven-3.8.4-bin.tar.gz

tritonserver/cppbuild.sh

@@ -7,6 +7,39 @@ if [[ -z "$PLATFORM" ]]; then
     exit
 fi
 
+INCLUDE_DEVELOPER_TOOLS_SERVER=${INCLUDE_DEVELOPER_TOOLS_SERVER:=0}
+
+if [[ ! -f "/opt/tritonserver/include/triton/developer_tools/generic_server_wrapper.h" ]] && [[ ! -f "/opt/tritonserver/lib/libtritondevelopertoolsserver.so" ]] && [[ ${INCLUDE_DEVELOPER_TOOLS_SERVER} -eq 0 ]]; then
+    TOOLS_BRANCH=${TOOLS_BRANCH:="https://github.com/triton-inference-server/developer_tools.git"}
+    TOOLS_BRANCH_TAG=${TOOLS_BRANCH_TAG:="main"}
+    TRITON_CORE_REPO=${TRITON_CORE_REPO:="https://github.com/triton-inference-server/core.git"}
+    TRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG="r23.05"}
+    TRITON_HOME="/opt/tritonserver"
+    BUILD_HOME="$PWD"/tritonbuild
+    mkdir -p ${BUILD_HOME} && cd ${BUILD_HOME}
+    git clone --single-branch --depth=1 -b ${TRITON_CORE_REPO_TAG} ${TRITON_CORE_REPO}
+    cp core/include/triton/core/* ${TRITON_HOME}/include/triton/core/.
+
+    # Build CPP library and copy over dependencies
+    git clone --single-branch --depth=1 -b ${TOOLS_BRANCH_TAG} ${TOOLS_BRANCH}
+    cd developer_tools/server
+    mkdir build && cd build
+    cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_BUILD_TEST=ON -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} -DTRITON_BUILD_STATIC_LIBRARY=OFF ..
+    make -j"$(grep -c ^processor /proc/cpuinfo)" install
+    # Copy dynamic library to triton home
+    cp ${BUILD_HOME}/developer_tools/server/build/install/lib/libtritondevelopertoolsserver.so ${TRITON_HOME}/lib/.
+    BUILD_INCLUDE_REPO=${BUILD_HOME}/developer_tools/server/include/triton/developer_tools
+    BUILD_SRC_REPO=${BUILD_HOME}/developer_tools/server/src
+    TRITON_INCLUDE_REPO=${TRITON_HOME}/include/triton/developer_tools
+    mkdir -p ${TRITON_INCLUDE_REPO}/src
+    cp ${BUILD_INCLUDE_REPO}/common.h ${TRITON_INCLUDE_REPO}/.
+    cp ${BUILD_INCLUDE_REPO}/generic_server_wrapper.h ${TRITON_INCLUDE_REPO}/.
+    cp ${BUILD_SRC_REPO}/infer_requested_output.h ${TRITON_INCLUDE_REPO}/src/.
+    cp ${BUILD_SRC_REPO}/tracer.h ${TRITON_INCLUDE_REPO}/src/.
+    cd ${BUILD_HOME}/..
+    rm -r ${BUILD_HOME}
+fi
+
 case $PLATFORM in
     linux-arm64)
         if [[ ! -f "/opt/tritonserver/include/triton/core/tritonserver.h" ]] && [[ ! -d "/opt/tritonserver/lib/" ]]; then

tritonserver/platform/pom.xml

@@ -6,13 +6,13 @@
   <parent>
     <groupId>org.bytedeco</groupId>
     <artifactId>javacpp-presets</artifactId>
-    <version>1.5.9</version>
+    <version>1.5.10-SNAPSHOT</version>
     <relativePath>../../</relativePath>
   </parent>
 
   <groupId>org.bytedeco</groupId>
   <artifactId>tritonserver-platform</artifactId>
-  <version>2.33-${project.parent.version}</version>
+  <version>2.34-${project.parent.version}</version>
   <name>JavaCPP Presets Platform for Triton Inference Server</name>
 
   <properties>

tritonserver/platform/redist/pom.xml

@@ -6,13 +6,13 @@
   <parent>
     <groupId>org.bytedeco</groupId>
     <artifactId>javacpp-presets</artifactId>
-    <version>1.5.9</version>
+    <version>1.5.10-SNAPSHOT</version>
     <relativePath>../../../</relativePath>
   </parent>
 
   <groupId>org.bytedeco</groupId>
   <artifactId>tritonserver-platform-redist</artifactId>
-  <version>2.33-${project.parent.version}</version>
+  <version>2.34-${project.parent.version}</version>
   <name>JavaCPP Presets Platform Redist for Triton Inference Server</name>
 
   <properties>

tritonserver/pom.xml

@@ -6,12 +6,12 @@
   <parent>
     <groupId>org.bytedeco</groupId>
     <artifactId>javacpp-presets</artifactId>
-    <version>1.5.9</version>
+    <version>1.5.10-SNAPSHOT</version>
   </parent>
 
   <groupId>org.bytedeco</groupId>
   <artifactId>tritonserver</artifactId>
-  <version>2.33-${project.parent.version}</version>
+  <version>2.34-${project.parent.version}</version>
   <name>JavaCPP Presets for Triton Inference Server</name>
 
   <dependencies>

tritonserver/samples/simple/Simple.java

@@ -370,6 +370,7 @@ output0, output1, new FloatPointer(input0_data), new FloatPointer(input1_data),
           "getting Triton API version");
       if ((TRITONSERVER_API_VERSION_MAJOR != api_version_major[0]) ||
           (TRITONSERVER_API_VERSION_MINOR > api_version_minor[0])) {
+        System.out.println("MAJOR " + api_version_major[0] + ", MINOR " + api_version_minor[0]);
         FAIL("triton server API version mismatch");
       }
 

tritonserver/samples/simple/pom.xml

@@ -2,7 +2,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.bytedeco.tritonserver</groupId>
     <artifactId>simple</artifactId>
-    <version>1.5.9</version>
+    <version>1.5.10-SNAPSHOT</version>
     <properties>
         <exec.mainClass>Simple</exec.mainClass>
         <maven.compiler.source>1.8</maven.compiler.source>
@@ -12,7 +12,7 @@
         <dependency>
             <groupId>org.bytedeco</groupId>
             <artifactId>tritonserver-platform</artifactId>
-            <version>2.33-1.5.9</version>
+            <version>2.34-1.5.10-SNAPSHOT</version>
             <classifier>shaded</classifier>
         </dependency>
     </dependencies>

tritonserver/samples/simplecpp/SimpleCPP.java

@@ -0,0 +1,166 @@
+// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+import com.google.gson.*;
+import org.bytedeco.javacpp.*;
+import org.bytedeco.tritonserver.tritondevelopertoolsserver.*;
+import static org.bytedeco.tritonserver.global.tritondevelopertoolsserver.*;
+
+public class SimpleCPP {
+    // Helper functions
+    static void FAIL(String MSG) {
+      System.err.println("Failure: " + MSG);
+      System.exit(1);
+    }
+
+    static void Usage(String msg) {
+      if (msg != null) {
+        System.err.println(msg);
+      }
+
+      System.err.println("Usage: java " + SimpleCPP.class.getSimpleName() +" [options]");
+      System.err.println("\t-v Enable verbose logging");
+      System.err.println("\t-r [model repository absolute path]");
+      System.exit(1);
+    }
+
+    static void
+    CompareResult(
+        String output0_name, String output1_name,
+        IntPointer input0, IntPointer input1, IntPointer output0,
+        IntPointer output1) {
+      for (int i = 0; i < 16; ++i) {
+        System.out.println(input0.get(i) + " + " + input1.get(i) + " = "
+                         + output0.get(i));
+        System.out.println(input0.get(i) + " - " + input1.get(i) + " = "
+                         + output1.get(i));
+
+        if ((input0.get(i) + input1.get(i)) != output0.get(i)) {
+          FAIL("incorrect sum in " + output0_name);
+        }
+        if ((input0.get(i) - input1.get(i)) != output1.get(i)) {
+          FAIL("incorrect difference in " + output1_name);
+        }
+      }
+    }
+
+    static void
+    GenerateInputData(
+        IntPointer[] input0_data, IntPointer[] input1_data) {
+      input0_data[0] = new IntPointer(16);
+      input1_data[0] = new IntPointer(16);
+      for (int i = 0; i < 16; ++i) {
+        input0_data[0].put(i, 2);
+        input1_data[0].put(i, 1 * i);
+      }
+    }
+
+    static int RunInference(int verbose_level, String model_repository_path, String model_name) {
+      StringVector model_repository_paths = new StringVector(model_repository_path);
+      ServerOptions options = new ServerOptions(model_repository_paths);
+      LoggingOptions logging_options = options.logging_();
+      logging_options.SetVerboseLevel(verbose_level);
+      options.SetLoggingOptions(logging_options);
+
+      GenericTritonServer server = GenericTritonServer.Create(options);
+      StringSet loaded_models = server.LoadedModels();
+      System.out.println("Loaded_models count : " + loaded_models.size());
+
+      InferOptions infer_options = new InferOptions(model_name);
+      GenericInferRequest request = GenericInferRequest.Create(infer_options);
+
+      BytePointer input0_data;
+      BytePointer input1_data;
+      IntPointer[] p0 = {null}, p1 = {null};
+      GenerateInputData(p0, p1);
+      input0_data = p0[0].getPointer(BytePointer.class);
+      input1_data = p1[0].getPointer(BytePointer.class);
+
+      LongPointer shape0 = new LongPointer(2);
+      LongPointer shape1 = new LongPointer(2);
+      shape0.put(0, 1);
+      shape0.put(1, 16);
+      shape1.put(0, 1);
+      shape1.put(1, 16);
+      Tensor tensor0 = new Tensor(input0_data, 4 * 16, 8, shape0, 0, 1);
+      Tensor tensor1 = new Tensor(input1_data, 4 * 16, 8, shape1, 0, 1);
+      request.AddInput("INPUT0", tensor0);
+      request.AddInput("INPUT1", tensor1);
+      GenericInferResult result = server.Infer(request);
+
+            Tensor output = result.Output("OUTPUT0");
+      BytePointer buffer = output.buffer_();
+
+      System.out.println("buffer to string : " + buffer.toString());
+      System.out.println("output val at index 0 : " + buffer.getInt(0));
+      System.out.println("output val at index 1 : " + buffer.getInt(1 * 4));
+      System.out.println("output val at index 2 : " + buffer.getInt(2 * 4));
+      System.out.println("output val at index 3 : " + buffer.getInt(3 * 4));
+      System.out.println("output val at index 4 : " + buffer.getInt(4 * 4));
+      System.out.println("output val at index 5 : " + buffer.getInt(5 * 4));
+      System.out.println("output val at index 6 : " + buffer.getInt(6 * 4));
+      System.out.println("output val at index 7 : " + buffer.getInt(7 * 4));
+      return 0;
+    }
+
+    public static void
+    main(String[] args) throws Exception {
+      String model_repository_path = "./models";
+      int verbose_level = 0;
+
+      // Parse commandline...
+      for (int i = 0; i < args.length; i++) {
+      switch (args[i]) {
+          case "-r":
+          model_repository_path = args[++i];
+          break;
+          case "-v":
+          verbose_level = 1;
+          break;
+          case "-?":
+          Usage(null);
+          break;
+        }
+      }
+
+      // We use a simple model that takes 2 input tensors of 16 strings
+      // each and returns 2 output tensors of 16 strings each. The input
+      // strings must represent integers. One output tensor is the
+      // element-wise sum of the inputs and one output is the element-wise
+      // difference.
+      String model_name = "simple";
+      if (model_repository_path == null) {
+        Usage("-r must be used to specify model repository path");
+      }
+
+      RunInference(verbose_level, model_repository_path, model_name);
+
+      System.exit(0);
+    }
+}

tritonserver/samples/simplecpp/pom.xml

@@ -0,0 +1,27 @@
+<project>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.bytedeco.tritonserver</groupId>
+    <artifactId>simplecpp</artifactId>
+    <version>1.5.10-SNAPSHOT</version>
+    <properties>
+        <exec.mainClass>SimpleCPP</exec.mainClass>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>org.bytedeco</groupId>
+            <artifactId>tritonserver-platform</artifactId>
+            <version>2.34-1.5.10-SNAPSHOT</version>
+            <classifier>shaded</classifier>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.9.0</version>
+        </dependency>
+    </dependencies>
+    <build>
+        <sourceDirectory>.</sourceDirectory>
+    </build>
+</project>